@ls-stack/agent-eval 0.19.0 → 0.21.0

package/dist/{runOrchestration-BBg_VUH5.mjs → runOrchestration-zYAcAPtS.mjs}

@@ -1970,7 +1970,15 @@ const numberDisplayOptionsSchema = z.object({
 	compactDisplay: z.enum(["short", "long"]).optional(),
 	prefix: z.string().optional(),
 	suffix: z.string().optional(),
-	decimalPlaces: z.number().int().min(0).optional()
+	minDecimalPlaces: z.number().int().min(0).optional(),
+	maxDecimalPlaces: z.number().int().min(0).optional()
+}).refine((options) => {
+	if (options.minDecimalPlaces === void 0) return true;
+	if (options.maxDecimalPlaces === void 0) return true;
+	return options.minDecimalPlaces <= options.maxDecimalPlaces;
+}, {
+	message: "minDecimalPlaces must be less than or equal to maxDecimalPlaces",
+	path: ["minDecimalPlaces"]
 });
 /** Schema for the supported column rendering kinds in list views. */
 const columnKindSchema = z.enum([
@@ -2005,7 +2013,6 @@ const columnDefSchema = z.object({
 	passThreshold: z.number().optional(),
 	maxStars: z.number().int().min(2).optional(),
 	hideInTable: z.boolean().optional(),
-	sortable: z.boolean().optional(),
 	align: z.enum([
 		"left",
 		"center",
@@ -2403,6 +2410,8 @@ const evalStatItemSchema = z.discriminatedUnion("kind", [
 		label: z.string().optional(),
 		aggregate: evalStatAggregateSchema,
 		format: columnFormatSchema.optional(),
+		/** Number presentation options applied when `format: 'number'`. */
+		numberFormat: numberDisplayOptionsSchema.optional(),
 		accent: z.boolean().optional()
 	})
 ]);
@@ -2566,6 +2575,21 @@ const caseDetailSchema = z.object({
 //#region ../shared/src/schemas/config.ts
 /** Strategy used to collapse repeated trials into one stored case result. */
 const trialSelectionModeSchema = z.enum(["lowestScore", "median"]);
+/** Built-in eval-level output/column keys. */
+const defaultConfigKeySchema = z.enum([
+	"apiCalls",
+	"costUsd",
+	"llmTurns",
+	"inputTokens",
+	"outputTokens",
+	"totalTokens",
+	"cachedInputTokens",
+	"cacheCreationInputTokens",
+	"reasoningTokens",
+	"llmLatencyMs"
+]);
+/** Removal config for built-in eval-level outputs and UI metadata. */
+const removeDefaultConfigSchema = z.union([z.literal(true), z.array(defaultConfigKeySchema)]);
 /** Render formats supported by an LLM-call metric in the UI. */
 const llmCallMetricFormatSchema = z.enum([
 	"string",
@@ -2637,6 +2661,30 @@ const apiCallMetricSchema = z.object({
 	*/
 	placements: z.array(apiCallMetricPlacementSchema).nonempty().optional()
 });
+/**
+* Schema for one model/provider pricing entry used to derive LLM-call costs
+* from token counts when a span does not already record explicit USD costs.
+*/
+const llmCallPricingSchema = z.object({
+	/** Exact model name read from the configured `attributes.model` path. */
+	model: z.string().min(1),
+	/**
+	* Optional provider discriminator read from `attributes.provider`. When set,
+	* the entry only applies to calls from that provider; provider-specific
+	* entries take precedence over generic entries for the same model.
+	*/
+	provider: z.string().min(1).optional(),
+	/** USD per one million non-cached input tokens. */
+	inputUsdPerMillion: z.number().nonnegative().optional(),
+	/** USD per one million output tokens. */
+	outputUsdPerMillion: z.number().nonnegative().optional(),
+	/** USD per one million prompt-cache read tokens. */
+	cachedInputUsdPerMillion: z.number().nonnegative().optional(),
+	/** USD per one million prompt-cache write tokens. */
+	cacheCreationInputUsdPerMillion: z.number().nonnegative().optional(),
+	/** USD per one million reasoning tokens when reported separately. */
+	reasoningUsdPerMillion: z.number().nonnegative().optional()
+});
 /** Schema for the global LLM calls config block in `agent-evals.config.ts`. */
 const llmCallsConfigSchema = z.object({
 	/** Span kinds treated as LLM calls. Defaults to `['llm']`. */
@@ -2647,8 +2695,9 @@ const llmCallsConfigSchema = z.object({
 	* built-in defaults (e.g. `usage.inputTokens`, `costUsd`).
 	*
 	* Per-token-type cost paths (`inputCost`, `outputCost`, `cachedInputCost`,
-	* `reasoningCost`) feed the cost breakdown table in the expanded row.
-	* Record them as USD numbers alongside `costUsd` in your span attributes.
+	* `reasoningCost`) feed the cost breakdown table in the expanded row when
+	* spans provide explicit USD cost overrides. Prefer `pricing` for deriving
+	* costs from token counts globally.
 	*/
 	attributes: z.object({
 		model: z.string().optional(),
@@ -2659,6 +2708,7 @@ const llmCallsConfigSchema = z.object({
 		cacheCreationInputTokens: z.string().optional(),
 		reasoningTokens: z.string().optional(),
 		totalTokens: z.string().optional(),
+		tokensPerSecond: z.string().optional(),
 		cost: z.string().optional(),
 		inputCost: z.string().optional(),
 		outputCost: z.string().optional(),
@@ -2672,6 +2722,12 @@ const llmCallsConfigSchema = z.object({
 		reasoning: z.string().optional(),
 		toolCalls: z.string().optional()
 	}).optional(),
+	/**
+	* Model/provider pricing registry used to calculate missing LLM-call costs
+	* from token counts. Explicit span attributes (`costUsd`, `cost.inputUsd`,
+	* etc.) take precedence over derived prices.
+	*/
+	pricing: z.array(llmCallPricingSchema).optional(),
 	/** Custom user-defined metrics surfaced on each LLM call. */
 	metrics: z.array(llmCallMetricSchema).optional()
 });
@@ -2719,6 +2775,7 @@ const DEFAULT_LLM_CALLS_CONFIG = {
 		cacheCreationInputTokens: "usage.cacheCreationInputTokens",
 		reasoningTokens: "usage.reasoningTokens",
 		totalTokens: "usage.totalTokens",
+		tokensPerSecond: "tokensPerSecond",
 		cost: "costUsd",
 		inputCost: "cost.inputUsd",
 		outputCost: "cost.outputUsd",
@@ -2732,7 +2789,8 @@ const DEFAULT_LLM_CALLS_CONFIG = {
 		reasoning: "reasoning",
 		toolCalls: "toolCalls"
 	},
-	metrics: []
+	metrics: [],
+	pricing: []
 };
 /** Default API-calls config the UI uses before the workspace fetch resolves. */
 const DEFAULT_API_CALLS_CONFIG = {
@@ -2765,6 +2823,8 @@ const DEFAULT_API_CALLS_CONFIG = {
 *   attribute path.
 * - Missing `metrics[].format` defaults to `'string'`.
 * - Missing `metrics[].placements` defaults to `['body']`.
+* - Missing `pricing` defaults to an empty registry; explicit span costs still
+*   take precedence over derived costs.
 */
 function resolveLlmCallsConfig(input) {
 	return {
@@ -2780,6 +2840,15 @@ function resolveLlmCallsConfig(input) {
 			format: m.format ?? "string",
 			numberFormat: m.numberFormat,
 			placements: m.placements ? [...m.placements] : ["body"]
+		})),
+		pricing: (input?.pricing ?? []).map((p) => ({
+			model: p.model,
+			provider: p.provider,
+			inputUsdPerMillion: p.inputUsdPerMillion,
+			outputUsdPerMillion: p.outputUsdPerMillion,
+			cachedInputUsdPerMillion: p.cachedInputUsdPerMillion,
+			cacheCreationInputUsdPerMillion: p.cacheCreationInputUsdPerMillion,
+			reasoningUsdPerMillion: p.reasoningUsdPerMillion
 		}))
 	};
 }
@@ -2821,6 +2890,7 @@ const agentEvalsConfigSchema = z.object({
 	allowCliRunAll: z.boolean().optional(),
 	traceDisplay: traceDisplayInputConfigSchema.optional(),
 	llmCalls: llmCallsConfigSchema.optional(),
+	removeDefaultConfig: removeDefaultConfigSchema.optional(),
 	apiCalls: apiCallsConfigSchema.optional(),
 	runLogs: runLogsConfigSchema.optional(),
 	cache: z.object({
@@ -3056,6 +3126,62 @@ function readString$2(attributes, path) {
 	const raw = getNestedAttribute(attributes, path);
 	return typeof raw === "string" && raw.length > 0 ? raw : null;
 }
+function computeTokenCost(tokens, usdPerMillion) {
+	if (tokens === null) return null;
+	if (tokens === 0) return 0;
+	if (usdPerMillion === void 0) return null;
+	return tokens / 1e6 * usdPerMillion;
+}
+function pickPricingEntry({ pricing, model, provider }) {
+	if (model === null) return null;
+	let fallback = null;
+	for (const entry of pricing) {
+		if (entry.model !== model) continue;
+		if (entry.provider === void 0) {
+			fallback ??= entry;
+			continue;
+		}
+		if (entry.provider === provider) return entry;
+	}
+	return fallback;
+}
+function computeFallbackTotalCost({ inputTokens, inputCostUsd, outputTokens, outputCostUsd, cachedInputTokens, cachedInputCostUsd, cacheCreationInputTokens, cacheCreationInputCostUsd, reasoningTokens, reasoningCostUsd }) {
+	const parts = [
+		{
+			tokens: inputTokens,
+			cost: inputCostUsd
+		},
+		{
+			tokens: outputTokens,
+			cost: outputCostUsd
+		},
+		{
+			tokens: cachedInputTokens,
+			cost: cachedInputCostUsd
+		},
+		{
+			tokens: cacheCreationInputTokens,
+			cost: cacheCreationInputCostUsd
+		},
+		{
+			tokens: reasoningTokens,
+			cost: reasoningCostUsd
+		}
+	];
+	let total = 0;
+	let hasCost = false;
+	let hasReportedTokens = false;
+	for (const part of parts) {
+		if (part.tokens === null) continue;
+		hasReportedTokens = true;
+		if (part.tokens === 0) continue;
+		if (part.cost === null) return null;
+		total += part.cost;
+		hasCost = true;
+	}
+	if (hasCost) return total;
+	return hasReportedTokens ? 0 : null;
+}
 function computeLatencyMs$1(span) {
 	if (span.endedAt === null) return null;
 	const started = Date.parse(span.startedAt);
@@ -3100,9 +3226,11 @@ function pickError$1(span) {
 * shape consumed by the LLM calls tab.
 *
 * Spans whose `kind` is not in `config.kinds` are dropped. Structured fields
-* (`model`, token counts, cost, etc.) are read via `getNestedAttribute` from
-* the configured paths, with safe coercion to `string | null` / `number |
-* null`. `totalTokens` falls back to a sum of input + output + cached when no
+* (`model`, token counts, explicit cost, etc.) are read via
+* `getNestedAttribute` from the configured paths, with safe coercion to
+* `string | null` / `number | null`. When explicit USD costs are absent,
+* configured model pricing derives per-token-type costs from token counts.
+* `totalTokens` falls back to a sum of input + output + cached when no
 * explicit total attribute is present. The `steps` attribute path may resolve
 * to either a number (rendered as the inference-round count) or an array of
 * per-step detail objects (rendered as a Steps section in the body, with
@@ -3118,12 +3246,36 @@ function extractLlmCalls(spans, config) {
 	for (const span of spans) {
 		if (!kindSet.has(span.kind)) continue;
 		const attrs = span.attributes;
+		const model = readString$2(attrs, config.attributes.model);
+		const provider = readString$2(attrs, config.attributes.provider);
 		const inputTokens = readNumber$2(attrs, config.attributes.inputTokens);
 		const outputTokens = readNumber$2(attrs, config.attributes.outputTokens);
 		const cachedInputTokens = readNumber$2(attrs, config.attributes.cachedInputTokens);
 		const cacheCreationInputTokens = readNumber$2(attrs, config.attributes.cacheCreationInputTokens);
 		const reasoningTokens = readNumber$2(attrs, config.attributes.reasoningTokens);
 		const declaredTotalTokens = readNumber$2(attrs, config.attributes.totalTokens);
+		const pricing = pickPricingEntry({
+			pricing: config.pricing,
+			model,
+			provider
+		});
+		const inputCostUsd = readNumber$2(attrs, config.attributes.inputCost) ?? computeTokenCost(inputTokens, pricing?.inputUsdPerMillion);
+		const outputCostUsd = readNumber$2(attrs, config.attributes.outputCost) ?? computeTokenCost(outputTokens, pricing?.outputUsdPerMillion);
+		const cachedInputCostUsd = readNumber$2(attrs, config.attributes.cachedInputCost) ?? computeTokenCost(cachedInputTokens, pricing?.cachedInputUsdPerMillion);
+		const cacheCreationInputCostUsd = readNumber$2(attrs, config.attributes.cacheCreationInputCost) ?? computeTokenCost(cacheCreationInputTokens, pricing?.cacheCreationInputUsdPerMillion);
+		const reasoningCostUsd = readNumber$2(attrs, config.attributes.reasoningCost) ?? computeTokenCost(reasoningTokens, pricing?.reasoningUsdPerMillion);
+		const costUsd = readNumber$2(attrs, config.attributes.cost) ?? computeFallbackTotalCost({
+			inputTokens,
+			inputCostUsd,
+			outputTokens,
+			outputCostUsd,
+			cachedInputTokens,
+			cachedInputCostUsd,
+			cacheCreationInputTokens,
+			cacheCreationInputCostUsd,
+			reasoningTokens,
+			reasoningCostUsd
+		});
 		const metrics = [];
 		for (const metric of config.metrics) {
 			const rawValue = getNestedAttribute(attrs, metric.path);
@@ -3142,8 +3294,8 @@ function extractLlmCalls(spans, config) {
 			name: span.name,
 			kind: span.kind,
 			status: span.status,
-			model: readString$2(attrs, config.attributes.model),
-			provider: readString$2(attrs, config.attributes.provider),
+			model,
+			provider,
 			inputTokens,
 			outputTokens,
 			cachedInputTokens,
@@ -3156,12 +3308,13 @@ function extractLlmCalls(spans, config) {
 				cached: cachedInputTokens,
 				cacheCreation: cacheCreationInputTokens
 			}),
-			costUsd: readNumber$2(attrs, config.attributes.cost),
-			inputCostUsd: readNumber$2(attrs, config.attributes.inputCost),
-			outputCostUsd: readNumber$2(attrs, config.attributes.outputCost),
-			cachedInputCostUsd: readNumber$2(attrs, config.attributes.cachedInputCost),
-			cacheCreationInputCostUsd: readNumber$2(attrs, config.attributes.cacheCreationInputCost),
-			reasoningCostUsd: readNumber$2(attrs, config.attributes.reasoningCost),
+			tokensPerSecond: readNumber$2(attrs, config.attributes.tokensPerSecond),
+			costUsd,
+			inputCostUsd,
+			outputCostUsd,
+			cachedInputCostUsd,
+			cacheCreationInputCostUsd,
+			reasoningCostUsd,
 			...readSteps(attrs, config.attributes.steps),
 			finishReason: readString$2(attrs, config.attributes.finishReason),
 			latencyMs: computeLatencyMs$1(span),
@@ -3792,6 +3945,80 @@ function isRecordLike(value) {
 	return typeof value === "object" && value !== null && !Array.isArray(value);
 }
 //#endregion
+//#region ../runner/src/chartValidation.ts
+function isValidColumnMetric(metric, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(metric.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart metric references unknown column "${metric.key}" — dropped`);
+		return false;
+	}
+	if (metric.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart metric "${metric.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function isValidTooltipExtra(extra, columnsByKey, evalId, warnings) {
+	const columnDef = columnsByKey.get(extra.key);
+	if (!columnDef) {
+		warnings.push(`[${evalId}] chart tooltip extra references unknown column "${extra.key}" — dropped`);
+		return false;
+	}
+	if (extra.aggregate === "passThresholdRate") {
+		if (columnDef.isScore !== true || typeof columnDef.passThreshold !== "number") {
+			warnings.push(`[${evalId}] chart tooltip extra "${extra.key}" uses "passThresholdRate" but the column is not a score with passThreshold — dropped`);
+			return false;
+		}
+	}
+	return true;
+}
+function sanitizeChart(chart, columnsByKey, evalId, warnings) {
+	const metrics = chart.metrics.filter((metric) => {
+		if (metric.source === "builtin") return true;
+		return isValidColumnMetric(metric, columnsByKey, evalId, warnings);
+	});
+	if (metrics.length === 0) {
+		warnings.push(`[${evalId}] chart had no valid metrics after validation — chart dropped`);
+		return null;
+	}
+	const tooltipExtras = chart.tooltipExtras?.filter((extra) => {
+		if (extra.source === "builtin") return true;
+		return isValidTooltipExtra(extra, columnsByKey, evalId, warnings);
+	});
+	return {
+		...chart,
+		metrics,
+		tooltipExtras: tooltipExtras?.length ? tooltipExtras : void 0
+	};
+}
+/**
+* Validate and sanitize an authored `charts` config against the eval's
+* declared columns. Drops metrics/extras that reference unknown columns or
+* misuse `passThresholdRate`, and drops entire charts whose metrics are all
+* invalid. Returns `charts: undefined` when nothing valid remains so the UI
+* falls back to rendering no chart (matching the opt-in default).
+*/
+function validateCharts(params) {
+	const { charts, columnDefs, evalId } = params;
+	if (!charts || charts.length === 0) return {
+		charts: void 0,
+		warnings: []
+	};
+	const columnsByKey = new Map(columnDefs.map((def) => [def.key, def]));
+	const warnings = [];
+	const sanitized = [];
+	for (const chart of charts) {
+		const result = sanitizeChart(chart, columnsByKey, evalId, warnings);
+		if (result) sanitized.push(result);
+	}
+	return {
+		charts: sanitized.length > 0 ? sanitized : void 0,
+		warnings
+	};
+}
+//#endregion
 //#region ../runner/src/columnBuilder.ts
 /**
 * Normalize a user-provided score definition (either a function or an
@@ -3817,7 +4044,6 @@ function getScoreOverride(def) {
 		format: def.format,
 		numberFormat: def.numberFormat,
 		hideInTable: def.hideInTable,
-		sortable: def.sortable,
 		align: def.align,
 		maxStars: def.maxStars
 	};
@@ -3830,7 +4056,6 @@ function mergeOverrides(base, override) {
 		format: override.format ?? base.format,
 		numberFormat: override.numberFormat ?? base.numberFormat,
 		hideInTable: override.hideInTable ?? base.hideInTable,
-		sortable: override.sortable ?? base.sortable,
 		align: override.align ?? base.align,
 		maxStars: override.maxStars ?? base.maxStars
 	};
@@ -3945,7 +4170,6 @@ function createColumnDef(params) {
 	if (override?.numberFormat !== void 0) def.numberFormat = override.numberFormat;
 	if (override?.maxStars !== void 0) def.maxStars = override.maxStars;
 	if (override?.hideInTable !== void 0) def.hideInTable = override.hideInTable;
-	if (override?.sortable !== void 0) def.sortable = override.sortable;
 	if (override?.align !== void 0) def.align = override.align;
 	if (!isScore) return def;
 	def.isScore = true;
@@ -4003,6 +4227,294 @@ async function loadConfig() {
 	}
 }
 //#endregion
+//#region ../runner/src/defaultConfig.ts
+const DEFAULT_CONFIG_KEYS = [
+	"apiCalls",
+	"costUsd",
+	"llmTurns",
+	"inputTokens",
+	"outputTokens",
+	"totalTokens",
+	"cachedInputTokens",
+	"cacheCreationInputTokens",
+	"llmLatencyMs"
+];
+const tokenNumberFormat = { notation: "compact" };
+const countNumberFormat = {
+	minDecimalPlaces: 0,
+	maxDecimalPlaces: 0
+};
+const costNumberFormat = {
+	prefix: "$",
+	maxDecimalPlaces: 4
+};
+const DEFAULT_COLUMNS = {
+	apiCalls: {
+		label: "API Calls",
+		format: "number",
+		numberFormat: countNumberFormat,
+		align: "right"
+	},
+	costUsd: {
+		label: "Cost",
+		format: "number",
+		numberFormat: costNumberFormat,
+		align: "right"
+	},
+	llmTurns: {
+		label: "LLM Turns",
+		format: "number",
+		numberFormat: countNumberFormat,
+		align: "right"
+	},
+	inputTokens: {
+		label: "Input Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	outputTokens: {
+		label: "Output Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	totalTokens: {
+		label: "Total Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	cachedInputTokens: {
+		label: "Cached Input Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	cacheCreationInputTokens: {
+		label: "Cache Write Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	reasoningTokens: {
+		label: "Reasoning Tokens",
+		format: "number",
+		numberFormat: tokenNumberFormat,
+		align: "right"
+	},
+	llmLatencyMs: {
+		label: "LLM Latency",
+		format: "duration",
+		align: "right"
+	}
+};
+function resolveRemovedKeys(globalRemove, evalRemove) {
+	if (globalRemove === true || evalRemove === true) return new Set(DEFAULT_CONFIG_KEYS);
+	return new Set([...globalRemove ?? [], ...evalRemove ?? []]);
+}
+function getActiveDefaultConfigKeys(params) {
+	const removed = resolveRemovedKeys(params.globalRemove, params.evalRemove);
+	return DEFAULT_CONFIG_KEYS.filter((key) => !removed.has(key));
+}
+function mergeDefaultColumns(params) {
+	const activeKeys = getActiveDefaultConfigKeys(params);
+	if (activeKeys.length === 0) return params.columns;
+	return {
+		...Object.fromEntries(activeKeys.map((key) => [key, DEFAULT_COLUMNS[key]])),
+		...params.columns
+	};
+}
+function appendDefaultStats(params) {
+	const activeKeys = new Set(getActiveDefaultConfigKeys(params));
+	const defaults = [];
+	if (activeKeys.has("apiCalls")) defaults.push({
+		kind: "column",
+		key: "apiCalls",
+		label: "API Calls",
+		aggregate: "avg",
+		numberFormat: countNumberFormat
+	});
+	if (activeKeys.has("costUsd")) defaults.push({
+		kind: "column",
+		key: "costUsd",
+		label: "LLM Cost",
+		aggregate: "avg",
+		numberFormat: costNumberFormat
+	});
+	if (activeKeys.has("totalTokens")) defaults.push({
+		kind: "column",
+		key: "totalTokens",
+		label: "Tokens",
+		aggregate: "avg",
+		numberFormat: tokenNumberFormat
+	});
+	if (activeKeys.has("llmTurns")) defaults.push({
+		kind: "column",
+		key: "llmTurns",
+		label: "LLM Turns",
+		aggregate: "avg",
+		numberFormat: countNumberFormat
+	});
+	const merged = [...params.stats ?? [], ...defaults];
+	return merged.length > 0 ? merged : void 0;
+}
+function appendDefaultCharts(params) {
+	const activeKeys = new Set(getActiveDefaultConfigKeys(params));
+	const defaults = [];
+	if (activeKeys.has("costUsd")) defaults.push({
+		heading: "LLM Cost",
+		type: "area",
+		metrics: [{
+			source: "column",
+			key: "costUsd",
+			aggregate: "avg",
+			label: "Cost",
+			color: "warning"
+		}]
+	});
+	const tokenMetrics = [
+		activeKeys.has("inputTokens") ? {
+			source: "column",
+			key: "inputTokens",
+			aggregate: "avg",
+			label: "Input",
+			color: "accent"
+		} : null,
+		activeKeys.has("outputTokens") ? {
+			source: "column",
+			key: "outputTokens",
+			aggregate: "avg",
+			label: "Output",
+			color: "success"
+		} : null,
+		activeKeys.has("cachedInputTokens") ? {
+			source: "column",
+			key: "cachedInputTokens",
+			aggregate: "avg",
+			label: "Cached Input",
+			color: "error"
+		} : null,
+		activeKeys.has("cacheCreationInputTokens") ? {
+			source: "column",
+			key: "cacheCreationInputTokens",
+			aggregate: "avg",
+			label: "Cache Write",
+			color: "warning"
+		} : null
+	].filter((metric) => metric !== null);
+	if (tokenMetrics.length > 0) defaults.push({
+		heading: "LLM Tokens",
+		type: "bar",
+		metrics: tokenMetrics,
+		tooltipExtras: activeKeys.has("totalTokens") ? [{
+			source: "column",
+			key: "totalTokens",
+			aggregate: "avg",
+			label: "Total"
+		}] : void 0
+	});
+	const merged = [...params.charts ?? [], ...defaults];
+	return merged.length > 0 ? merged : void 0;
+}
+function resolveEvalDefaultConfig(params) {
+	const evalRemove = params.evalDef.removeDefaultConfig;
+	return {
+		columns: mergeDefaultColumns({
+			columns: params.evalDef.columns,
+			globalRemove: params.globalRemove,
+			evalRemove
+		}),
+		stats: appendDefaultStats({
+			stats: params.evalDef.stats,
+			globalRemove: params.globalRemove,
+			evalRemove
+		}),
+		charts: appendDefaultCharts({
+			charts: params.evalDef.charts,
+			globalRemove: params.globalRemove,
+			evalRemove
+		})
+	};
+}
+function sumNullable(values) {
+	let total = 0;
+	let hasValue = false;
+	for (const value of values) {
+		if (value === null) continue;
+		total += value;
+		hasValue = true;
+	}
+	return hasValue ? total : void 0;
+}
+function assignIfMissing(params) {
+	if (!params.activeKeys.has(params.key)) return;
+	if (params.key in params.outputs) return;
+	if (params.value === void 0) return;
+	params.outputs[params.key] = params.value;
+}
+function addDefaultOutputs(params) {
+	const activeKeys = new Set(getActiveDefaultConfigKeys(params));
+	if (activeKeys.size === 0) return;
+	const calls = extractLlmCalls(params.spans, params.llmCallsConfig);
+	const apiCalls = extractApiCalls(params.spans, params.apiCallsConfig);
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "apiCalls",
+		value: apiCalls.length > 0 ? apiCalls.length : void 0,
+		activeKeys
+	});
+	if (calls.length === 0) return;
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "llmTurns",
+		value: calls.length,
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "costUsd",
+		value: sumNullable(calls.map((call) => call.costUsd)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "inputTokens",
+		value: sumNullable(calls.map((call) => call.inputTokens)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "outputTokens",
+		value: sumNullable(calls.map((call) => call.outputTokens)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "totalTokens",
+		value: sumNullable(calls.map((call) => call.totalTokens)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "cachedInputTokens",
+		value: sumNullable(calls.map((call) => call.cachedInputTokens)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "cacheCreationInputTokens",
+		value: sumNullable(calls.map((call) => call.cacheCreationInputTokens)),
+		activeKeys
+	});
+	assignIfMissing({
+		outputs: params.outputs,
+		key: "llmLatencyMs",
+		value: sumNullable(calls.map((call) => call.latencyMs)),
+		activeKeys
+	});
+}
+//#endregion
 //#region ../runner/src/discovery.ts
 const evalIdMatchRegex = /\bid\s*:\s*['"]([^'"]+)['"]/;
 const evalTitleMatchRegex = /\btitle\s*:\s*['"]([^'"]+)['"]/;
@@ -4664,7 +5176,7 @@ async function callWithUnknownResult(fn, args) {
 	return await Reflect.apply(fn, void 0, args);
 }
 async function runCase(params) {
-	const { evalDef, evalId, evalCase, globalTraceDisplay, trial, startTime, cacheAdapter, cacheMode, codeFingerprint, moduleIsolation, evalFilePath, workspaceRoot, artifactDir, runId } = params;
+	const { evalDef, evalId, evalCase, globalTraceDisplay, llmCallsConfig = resolveLlmCallsConfig(void 0), apiCallsConfig = resolveApiCallsConfig(void 0), globalRemoveDefaultConfig, trial, startTime, cacheAdapter, cacheMode, codeFingerprint, moduleIsolation, evalFilePath, workspaceRoot, artifactDir, runId } = params;
 	const scopedIdPrefix = buildScopedEvalIdPrefix({
 		evalId,
 		evalFilePath,
@@ -4714,6 +5226,14 @@ async function runCase(params) {
 			scope.assertionFailures.push(toAssertionFailure(message, e instanceof Error ? e : void 0));
 		}
 	}
+	if (!nonAssertError) addDefaultOutputs({
+		outputs: scope.outputs,
+		spans: scope.spans,
+		llmCallsConfig,
+		apiCallsConfig,
+		globalRemove: globalRemoveDefaultConfig,
+		evalRemove: evalDef.removeDefaultConfig
+	});
 	if (!nonAssertError && evalDef.outputsSchema) {
 		const { outputsSchema } = evalDef;
 		const parsedOutputs = await runInExistingEvalScope(scope, "outputsSchema", () => outputsSchema.safeParse(getOutputsSchemaInput(outputsSchema, scope.outputs)));
@@ -4795,6 +5315,11 @@ async function runCase(params) {
 	const status = nonAssertError ? "error" : passed ? "pass" : "fail";
 	const { trace: displayTrace, traceDisplay } = resolveTracePresentation(scope.spans, globalTraceDisplay, evalDef.traceDisplay);
 	const columns = {};
+	const columnOverrides = mergeDefaultColumns({
+		columns: evalDef.columns,
+		globalRemove: globalRemoveDefaultConfig,
+		evalRemove: evalDef.removeDefaultConfig
+	});
 	for (const [key, value] of Object.entries(scope.outputs)) {
 		const cell = isBlob(value) ? await persistInlineArtifact({
 			artifactDir,
@@ -4803,7 +5328,7 @@ async function runCase(params) {
 			outputKey: key,
 			trial,
 			value
-		}) : toCellValue(value, evalDef.columns?.[key]);
+		}) : toCellValue(value, columnOverrides?.[key]);
 		if (cell !== void 0) columns[key] = cell;
 	}
 	for (const key of Object.keys(evalDef.manualScores ?? {})) columns[key] = null;
@@ -5016,6 +5541,8 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 			key: runState.manifest.id,
 			workspaceRoot
 		};
+		const llmCallsConfig = resolveLlmCallsConfig(config.llmCalls);
+		const apiCallsConfig = resolveApiCallsConfig(config.apiCalls);
 		for (const evalMeta of targetEvals) {
 			const evalFilePath = evalMeta.sourceFilePath;
 			let codeFingerprint = "";
@@ -5054,7 +5581,20 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 								evalId: evalMeta.id
 							}), request.target.evalIds, request.target.caseIds, evalMeta.id);
 							runState.summary.totalCases += cases.length;
-							const accumulatedColumns = /* @__PURE__ */ new Map();
+							const defaultConfig = resolveEvalDefaultConfig({
+								evalDef,
+								globalRemove: config.removeDefaultConfig
+							});
+							const declaredColumnDefs = buildDeclaredColumnDefs(defaultConfig.columns, evalDef.scores, evalDef.manualScores);
+							const accumulatedColumns = new Map(declaredColumnDefs.map((def) => [def.key, def]));
+							const validatedCharts = validateCharts({
+								charts: defaultConfig.charts,
+								columnDefs: declaredColumnDefs,
+								evalId: evalMeta.id
+							});
+							for (const warning of validatedCharts.warnings) console.warn(warning);
+							evalMeta.stats = defaultConfig.stats;
+							evalMeta.charts = validatedCharts.charts;
 							const evalCaseRows = [];
 							const preparedCases = [];
 							const scoreKeys = Object.freeze(Object.keys(evalDef.scores ?? {}));
@@ -5066,7 +5606,7 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 								preparedCases,
 								scoreKeys: Object.freeze([...scoreKeys, ...manualScoreKeys]),
 								mergeColumns: (columns) => {
-									mergeColumnDefs(accumulatedColumns, columns, evalDef.columns, evalDef.scores, evalDef.manualScores);
+									mergeColumnDefs(accumulatedColumns, columns, defaultConfig.columns, evalDef.scores, evalDef.manualScores);
 								}
 							};
 							preparedEvals.push(preparedEval);
@@ -5087,6 +5627,9 @@ async function executeRun({ runState, request, runDir, config, evals, cacheStore
 												evalId: evalMeta.id,
 												evalCase,
 												globalTraceDisplay,
+												llmCallsConfig,
+												apiCallsConfig,
+												globalRemoveDefaultConfig: config.removeDefaultConfig,
 												trial,
 												startTime,
 												cacheAdapter: bufferedCacheStore ?? (cacheEnabled ? cacheStore : null),
@@ -5237,4 +5780,4 @@ function toLastRunStatus(status) {
 	return status === "pending" ? null : status;
 }
 //#endregion
-export { assertionFailureSchema as $, runArtifactRefSchema as $t, getNestedAttribute as A, getEvalRegistry as An, cacheRecordingSchema as At, agentEvalsConfigSchema as B, traceDisplayInputConfigSchema as Bt, createRunRequestSchema as C, runInEvalScope as Cn, cacheEntrySchema as Ct, extractCacheHits as D, startEvalBackgroundJob as Dn, cacheModeSchema as Dt, extractCacheEntries as E, setScopeCacheContext as En, cacheListItemSchema as Et, deriveStatusFromChildStatuses as F, traceAttributeDisplayFormatSchema as Ft, llmCallMetricFormatSchema as G, cellValueSchema as Gt, apiCallMetricPlacementSchema as H, traceSpanKindSchema as Ht, runManifestSchema as I, traceAttributeDisplayInputSchema as It, llmCallsConfigSchema as J, columnKindSchema as Jt, llmCallMetricPlacementSchema as K, columnDefSchema as Kt, runSummarySchema as L, traceAttributeDisplayPlacementSchema as Lt, getEvalDisplayStatus as M, serializedCacheSpanSchema as Mt, deriveScopedSummaryFromCases as N, spanCacheOptionsSchema as Nt, extractApiCalls as O, repoFile as On, cacheOperationTypeSchema as Ot, deriveStatusFromCaseRows as P, traceCacheRefSchema as Pt, trialSelectionModeSchema as Q, repoFileRefSchema as Qt, DEFAULT_API_CALLS_CONFIG as R, traceAttributeDisplaySchema as Rt, createFsCacheStore as S, runInEvalRuntimeScope as Sn, cacheDebugKeyFileSchema as St, sseEnvelopeSchema as T, setEvalOutput as Tn, cacheFileSchema as Tt, apiCallMetricSchema as U, traceSpanSchema as Ut, apiCallMetricFormatSchema as V, traceSpanErrorSchema as Vt, apiCallsConfigSchema as W, traceSpanWarningSchema as Wt, resolveLlmCallsConfig as X, jsonCellSchema as Xt, resolveApiCallsConfig as Y, fileRefSchema as Yt, runLogsConfigSchema as Z, numberDisplayOptionsSchema as Zt, loadEvalModule as _, getEvalCaseInput as _n, evalChartMetricSchema as _t, loadPersistedRunSnapshot as a, hashCacheKey as an, evalStatsConfigSchema as at, buildDeclaredColumnDefs as b, mergeEvalOutput as bn, evalChartsConfigSchema as bt, persistCaseDetail as c, deserializeCacheValue as cn, runLogLevelSchema as ct, recomputePersistedCaseStatus as d, EvalAssertionError as dn, scoreTraceSchema as dt, z$1 as en, caseDetailSchema as et, runTouchesEval as f, appendToEvalOutput as fn, evalChartAggregateSchema as ft, setLatestRunInfoMap as g, getCurrentScope as gn, evalChartConfigSchema as gt, getTargetEvalIds as h, evalLog as hn, evalChartColorSchema as ht, getLatestRunInfos as i, evalTracer as in, evalStatItemSchema as it, getEvalTitle as j, cacheStatusSchema as jt, extractLlmCalls as k, defineEval as kn, cacheRecordingOpSchema as kt, persistRunState as l, serializeCacheRecording as ln, runLogLocationSchema as lt, buildEvalSummary as m, evalAssert as mn, evalChartBuiltinMetricSchema as mt, generateRunId as n, captureEvalSpanError as nn, evalFreshnessStatusSchema as nt, loadPersistedRunSnapshots as o, hashCacheKeySync as on, evalSummarySchema as ot, resolveArtifactPath as p, configureEvalRunLogs as pn, evalChartAxisSchema as pt, llmCallMetricSchema as q, columnFormatSchema as qt, getLastRunStatuses as r, evalSpan as rn, evalStatAggregateSchema as rt, nextShortIdFromSnapshots as s, deserializeCacheRecording as sn, runLogEntrySchema as st, executeRun as t, buildTraceTree as tn, caseRowSchema as tt, recomputeEvalStatusesInRuns as u, serializeCacheValue as un, runLogPhaseSchema as ut, parseEvalMetas as v, incrementEvalOutput as vn, evalChartTooltipExtraSchema as vt, updateManualScoreRequestSchema as w, runInExistingEvalScope as wn, cacheEntryWithDebugKeySchema as wt, normalizeScoreDef as x, nextEvalId as xn, cacheDebugKeyEntrySchema as xt, loadConfig as y, isInEvalScope as yn, evalChartTypeSchema as yt, DEFAULT_LLM_CALLS_CONFIG as z, traceDisplayConfigSchema as zt };
+export { removeDefaultConfigSchema as $, columnKindSchema as $t, extractApiCalls as A, setEvalOutput as An, cacheFileSchema as At, DEFAULT_API_CALLS_CONFIG as B, traceAttributeDisplayFormatSchema as Bt, validateCharts as C, incrementEvalOutput as Cn, evalChartTooltipExtraSchema as Ct, sseEnvelopeSchema as D, runInEvalRuntimeScope as Dn, cacheDebugKeyFileSchema as Dt, updateManualScoreRequestSchema as E, nextEvalId as En, cacheDebugKeyEntrySchema as Et, deriveScopedSummaryFromCases as F, getEvalRegistry as Fn, cacheRecordingSchema as Ft, apiCallMetricSchema as G, traceDisplayInputConfigSchema as Gt, agentEvalsConfigSchema as H, traceAttributeDisplayPlacementSchema as Ht, deriveStatusFromCaseRows as I, cacheStatusSchema as It, llmCallMetricFormatSchema as J, traceSpanSchema as Jt, apiCallsConfigSchema as K, traceSpanErrorSchema as Kt, deriveStatusFromChildStatuses as L, serializedCacheSpanSchema as Lt, getNestedAttribute as M, startEvalBackgroundJob as Mn, cacheModeSchema as Mt, getEvalTitle as N, repoFile as Nn, cacheOperationTypeSchema as Nt, extractCacheEntries as O, runInEvalScope as On, cacheEntrySchema as Ot, getEvalDisplayStatus as P, defineEval as Pn, cacheRecordingOpSchema as Pt, llmCallsConfigSchema as Q, columnFormatSchema as Qt, runManifestSchema as R, spanCacheOptionsSchema as Rt, normalizeScoreDef as S, getEvalCaseInput as Sn, evalChartMetricSchema as St, createRunRequestSchema as T, mergeEvalOutput as Tn, evalChartsConfigSchema as Tt, apiCallMetricFormatSchema as U, traceAttributeDisplaySchema as Ut, DEFAULT_LLM_CALLS_CONFIG as V, traceAttributeDisplayInputSchema as Vt, apiCallMetricPlacementSchema as W, traceDisplayConfigSchema as Wt, llmCallMetricSchema as X, cellValueSchema as Xt, llmCallMetricPlacementSchema as Y, traceSpanWarningSchema as Yt, llmCallPricingSchema as Z, columnDefSchema as Zt, loadEvalModule as _, appendToEvalOutput as _n, evalChartAggregateSchema as _t, loadPersistedRunSnapshot as a, z$1 as an, caseDetailSchema as at, loadConfig as b, evalLog as bn, evalChartColorSchema as bt, persistCaseDetail as c, evalSpan as cn, evalStatAggregateSchema as ct, recomputePersistedCaseStatus as d, hashCacheKeySync as dn, evalSummarySchema as dt, fileRefSchema as en, resolveApiCallsConfig as et, runTouchesEval as f, deserializeCacheRecording as fn, runLogEntrySchema as ft, setLatestRunInfoMap as g, EvalAssertionError as gn, scoreTraceSchema as gt, getTargetEvalIds as h, serializeCacheValue as hn, runLogPhaseSchema as ht, getLatestRunInfos as i, runArtifactRefSchema as in, assertionFailureSchema as it, extractLlmCalls as j, setScopeCacheContext as jn, cacheListItemSchema as jt, extractCacheHits as k, runInExistingEvalScope as kn, cacheEntryWithDebugKeySchema as kt, persistRunState as l, evalTracer as ln, evalStatItemSchema as lt, buildEvalSummary as m, serializeCacheRecording as mn, runLogLocationSchema as mt, generateRunId as n, numberDisplayOptionsSchema as nn, runLogsConfigSchema as nt, loadPersistedRunSnapshots as o, buildTraceTree as on, caseRowSchema as ot, resolveArtifactPath as p, deserializeCacheValue as pn, runLogLevelSchema as pt, defaultConfigKeySchema as q, traceSpanKindSchema as qt, getLastRunStatuses as r, repoFileRefSchema as rn, trialSelectionModeSchema as rt, nextShortIdFromSnapshots as s, captureEvalSpanError as sn, evalFreshnessStatusSchema as st, executeRun as t, jsonCellSchema as tn, resolveLlmCallsConfig as tt, recomputeEvalStatusesInRuns as u, hashCacheKey as un, evalStatsConfigSchema as ut, parseEvalMetas as v, configureEvalRunLogs as vn, evalChartAxisSchema as vt, createFsCacheStore as w, isInEvalScope as wn, evalChartTypeSchema as wt, buildDeclaredColumnDefs as x, getCurrentScope as xn, evalChartConfigSchema as xt, resolveEvalDefaultConfig as y, evalAssert as yn, evalChartBuiltinMetricSchema as yt, runSummarySchema as z, traceCacheRefSchema as zt };