npm - @ls-stack/agent-eval - Versions diffs - 0.61.2 → 0.62.0 - Mend

@ls-stack/agent-eval 0.61.2 → 0.62.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (16) hide show

package/dist/{app-Dm_9ZTVa.mjs → app-CByWi7LX.mjs} +4 -4
package/dist/apps/web/dist/assets/{index-DXQ_LDQw.js → index-CE1teCsp.js} +1 -1
package/dist/apps/web/dist/index.html +1 -1
package/dist/bin.mjs +1 -1
package/dist/caseChild.mjs +1 -1
package/dist/{cli-CPBIcMP-.mjs → cli-CCHcjbC1.mjs} +4 -4
package/dist/index.d.mts +15 -3
package/dist/index.mjs +3 -3
package/dist/runChild.mjs +2 -2
package/dist/{runExecution-D-CnSRYy.mjs → runExecution-C24aYsk3.mjs} +97 -15
package/dist/{runOrchestration-Basvyp4u.mjs → runOrchestration-9XKoYcP9.mjs} +1 -1
package/dist/{runner-B6UT1K7L.mjs → runner-5eU-FLHV.mjs} +1 -1
package/dist/{runner-DwNb5TCb.mjs → runner-BEQGkHF0.mjs} +2 -2
package/dist/{src-SixIk0b7.mjs → src-C8n7QANC.mjs} +2 -2
package/package.json +3 -3
package/skills/agent-eval/SKILL.md +2 -2

package/dist/apps/web/dist/index.html CHANGED Viewed

@@ -25,7 +25,7 @@
       href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600&family=JetBrains+Mono:wght@400;500&display=swap"
       rel="stylesheet"
     />
-    <script type="module" crossorigin src="/assets/index-DXQ_LDQw.js"></script>
+    <script type="module" crossorigin src="/assets/index-CE1teCsp.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-zWPuRQmP.css">
   </head>
   <body>

package/dist/bin.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { t as runCli } from "./cli-CPBIcMP-.mjs";
+import { t as runCli } from "./cli-CCHcjbC1.mjs";
 import { spawn } from "node:child_process";
 //#region src/bin.ts
 const moduleMocksFlag = "--experimental-test-module-mocks";

package/dist/caseChild.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-import { Ft as runWithEvalRegistry, I as configureEvalRunLogs, St as resolveLlmCallsConfig, _ as createFsCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, g as createBufferedCacheStore, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as getCacheRetentionOptions, xt as resolveApiCallsConfig } from "./runExecution-D-CnSRYy.mjs";
+import { Ft as runWithEvalRegistry, I as configureEvalRunLogs, St as resolveLlmCallsConfig, _ as createFsCacheStore, a as isCaseChildParentMessage, d as loadEvalModule, g as createBufferedCacheStore, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, q as runInEvalRuntimeScope, r as runCase, v as getCacheRetentionOptions, xt as resolveApiCallsConfig } from "./runExecution-C24aYsk3.mjs";
 //#region ../runner/src/caseChild.ts
 let fatalErrorReported = false;
 let disconnectExpected = false;

package/dist/{cli-CPBIcMP-.mjs → cli-CCHcjbC1.mjs} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { Dt as caseRowSchema, Pt as getEvalRegistry, St as resolveLlmCallsConfig, Tt as getCaseRowCaseKey, _ as createFsCacheStore, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as getCacheRetentionOptions, wt as buildEvalKey, xt as resolveApiCallsConfig } from "./runExecution-D-CnSRYy.mjs";
-import { C as parseEvalDiscovery, S as loadIsolatedEvalRegistry, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as parseManualInputValues, c as getLatestRunInfos, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, l as loadPersistedCaseDetail, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as generateRunId, p as persistCaseDetail, s as getLastRunStatuses, u as loadPersistedRunSnapshot, v as runTouchesEval, w as validateCharts, x as deriveEvalFreshness, y as buildManualInputDescriptor } from "./runOrchestration-Basvyp4u.mjs";
+import { Dt as caseRowSchema, Pt as getEvalRegistry, St as resolveLlmCallsConfig, Tt as getCaseRowCaseKey, _ as createFsCacheStore, bt as runSummarySchema, c as resolveArtifactPath, ct as applyDerivedCallAttributes, dt as getEvalDisplayStatus, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, m as buildDeclaredColumnDefs, o as stripTerminalControlCodes, p as loadConfig, s as resolveTracePresentation, ut as getEvalTitle, v as getCacheRetentionOptions, wt as buildEvalKey, xt as resolveApiCallsConfig } from "./runExecution-C24aYsk3.mjs";
+import { C as parseEvalDiscovery, S as loadIsolatedEvalRegistry, _ as recomputePersistedCaseStatus, a as validateTagsFilters, b as parseManualInputValues, c as getLatestRunInfos, d as loadPersistedRunSnapshots, f as nextShortIdFromSnapshots, g as recomputeEvalStatusesInRuns, h as persistRunState, i as resolveEvalTags, l as loadPersistedCaseDetail, m as deleteTemporaryRuns, n as getTargetEvalKeys, o as generateRunId, p as persistCaseDetail, s as getLastRunStatuses, u as loadPersistedRunSnapshot, v as runTouchesEval, w as validateCharts, x as deriveEvalFreshness, y as buildManualInputDescriptor } from "./runOrchestration-9XKoYcP9.mjs";
 import { parseEnv } from "node:util";
 import { resultify } from "t-result";
 import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
@@ -2243,8 +2243,8 @@ async function commandApp(args) {
 	const { serve } = await import("@hono/node-server");
 	const bundledWebDist = resolve(currentDir, "apps/web/dist");
 	if (existsSync(bundledWebDist)) process.env.AGENT_EVALS_WEB_DIST = bundledWebDist;
-	const appModule = await import("./app-Dm_9ZTVa.mjs");
-	const runnerModule = await import("./runner-B6UT1K7L.mjs");
+	const appModule = await import("./app-CByWi7LX.mjs");
+	const runnerModule = await import("./runner-5eU-FLHV.mjs");
 	if (!isHonoAppModule(appModule)) throw new Error("Server app module is invalid");
 	if (!isServerRunnerModule(runnerModule)) throw new Error("Server runner module is invalid");
 	await runnerModule.initRunner({ loadEnv: args.loadEnv });

package/dist/index.d.mts CHANGED Viewed

@@ -377,7 +377,12 @@ declare const evalChartsConfigSchema$1: z.ZodArray<z.ZodObject<{
 /** Ordered list of history charts rendered for an eval. */
 type EvalChartsConfig$1 = z.infer<typeof evalChartsConfigSchema$1>; //#endregion
 //#region ../shared/src/schemas/config.d.ts
-/** Built-in eval-level output/column keys. */
+/**
+ * Built-in eval-level output/column keys.
+ *
+ * `costUsd` controls the default LLM cost family: actual billed cost plus the
+ * normalized `costUsdWithoutCache` and `costUsdWarmedCache` chart outputs.
+ */
 declare const defaultConfigKeySchema: z.ZodEnum<{
   apiCalls: "apiCalls";
   costUsd: "costUsd";
@@ -3684,7 +3689,12 @@ declare const trialSelectionModeSchema: z.ZodEnum<{
 }>;
 /** Strategy used to collapse repeated trials into one stored case result. */
 type TrialSelectionMode = z.infer<typeof trialSelectionModeSchema>;
-/** Built-in eval-level output/column keys. */
+/**
+ * Built-in eval-level output/column keys.
+ *
+ * `costUsd` controls the default LLM cost family: actual billed cost plus the
+ * normalized `costUsdWithoutCache` and `costUsdWarmedCache` chart outputs.
+ */
 /** Removal config for built-in eval-level outputs and UI metadata. */
 declare const removeDefaultConfigSchema: z.ZodUnion<readonly [z.ZodLiteral<true>, z.ZodArray<z.ZodEnum<{
   costUsd: "costUsd";
@@ -4285,7 +4295,9 @@ type AgentEvalsConfig$1 = {
    * Defaults are derived from trace spans using the resolved `llmCalls` and
    * `apiCalls` extraction configs. Set to `true` to remove all defaults, or
    * pass specific keys such as `['costUsd', 'apiCalls']` to remove only those
-   * defaults globally. Per-eval removal is additive.
+   * defaults globally. Removing `costUsd` removes the whole default cost
+   * family, including normalized no-cache and warmed-cache outputs. Per-eval
+   * removal is additive.
    */
   removeDefaultConfig?: RemoveDefaultConfig;
   /**

package/dist/index.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-import { $ as startEvalBackgroundJob, A as manualInputFileValueSchema, B as getCurrentScope, C as hashCacheKey, D as serializeCacheRecording, E as deserializeCacheValue, F as appendToEvalOutput, G as mergeEvalOutput, H as incrementEvalOutput, J as runInEvalScope, K as nextEvalId, L as evalAssert, M as evalExpect, N as EvalAssertionError, O as serializeCacheValue, P as EvalRuntimeUsageError, Pt as getEvalRegistry, Q as setScopeCacheContext, R as evalLog, S as evalTracer, T as deserializeCacheRecording, U as isInEvalScope, V as getEvalCaseInput, Y as runInExistingEvalScope, Z as setEvalOutput, at as extractLlmCalls, b as captureEvalSpanError, it as extractApiCalls, j as readManualInputFile, k as repoFile, lt as getNestedAttribute, nt as extractCacheEntries, ot as simulateLlmCallCost, q as runInEvalRuntimeScope, rt as extractCacheHits, st as simulateTokenAllocation, w as hashCacheKeySync, x as evalSpan, y as buildTraceTree, z as evalTime } from "./runExecution-D-CnSRYy.mjs";
-import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-CPBIcMP-.mjs";
-import { n as matchesEvalTags, t as defineEval } from "./src-SixIk0b7.mjs";
+import { $ as startEvalBackgroundJob, A as manualInputFileValueSchema, B as getCurrentScope, C as hashCacheKey, D as serializeCacheRecording, E as deserializeCacheValue, F as appendToEvalOutput, G as mergeEvalOutput, H as incrementEvalOutput, J as runInEvalScope, K as nextEvalId, L as evalAssert, M as evalExpect, N as EvalAssertionError, O as serializeCacheValue, P as EvalRuntimeUsageError, Pt as getEvalRegistry, Q as setScopeCacheContext, R as evalLog, S as evalTracer, T as deserializeCacheRecording, U as isInEvalScope, V as getEvalCaseInput, Y as runInExistingEvalScope, Z as setEvalOutput, at as extractLlmCalls, b as captureEvalSpanError, it as extractApiCalls, j as readManualInputFile, k as repoFile, lt as getNestedAttribute, nt as extractCacheEntries, ot as simulateLlmCallCost, q as runInEvalRuntimeScope, rt as extractCacheHits, st as simulateTokenAllocation, w as hashCacheKeySync, x as evalSpan, y as buildTraceTree, z as evalTime } from "./runExecution-C24aYsk3.mjs";
+import { a as materializeManualInputFiles, i as isManualInputFileValue, n as createRunner, o as stageManualInputFile, r as cleanupStagedManualInputFiles, s as stageManualInputFileFromPath, t as runCli } from "./cli-CCHcjbC1.mjs";
+import { n as matchesEvalTags, t as defineEval } from "./src-C8n7QANC.mjs";
 export { EvalAssertionError, EvalRuntimeUsageError, appendToEvalOutput, buildTraceTree, captureEvalSpanError, cleanupStagedManualInputFiles, createRunner, defineEval, deserializeCacheRecording, deserializeCacheValue, evalAssert, evalExpect, evalLog, evalSpan, evalTime, evalTracer, extractApiCalls, extractCacheEntries, extractCacheHits, extractLlmCalls, getCurrentScope, getEvalCaseInput, getEvalRegistry, getNestedAttribute, hashCacheKey, hashCacheKeySync, incrementEvalOutput, isInEvalScope, isManualInputFileValue, manualInputFileValueSchema, matchesEvalTags, materializeManualInputFiles, mergeEvalOutput, nextEvalId, readManualInputFile, repoFile, runCli, runInEvalRuntimeScope, runInEvalScope, runInExistingEvalScope, serializeCacheRecording, serializeCacheValue, setEvalOutput, setScopeCacheContext, simulateLlmCallCost, simulateTokenAllocation, stageManualInputFile, stageManualInputFileFromPath, startEvalBackgroundJob };

package/dist/runChild.mjs CHANGED Viewed

@@ -1,5 +1,5 @@
-import { At as manualInputDescriptorSchema, I as configureEvalRunLogs, Mt as columnDefSchema, Ot as evalStatAggregateSchema, _ as createFsCacheStore, bt as runSummarySchema, et as createRunRequestSchema, jt as evalChartsConfigSchema, kt as evalStatsConfigSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, v as getCacheRetentionOptions, wt as buildEvalKey, yt as runManifestSchema } from "./runExecution-D-CnSRYy.mjs";
-import { C as parseEvalDiscovery, h as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-Basvyp4u.mjs";
+import { At as manualInputDescriptorSchema, I as configureEvalRunLogs, Mt as columnDefSchema, Ot as evalStatAggregateSchema, _ as createFsCacheStore, bt as runSummarySchema, et as createRunRequestSchema, jt as evalChartsConfigSchema, kt as evalStatsConfigSchema, l as registerAgentEvalsPackageResolutionHooks, p as loadConfig, v as getCacheRetentionOptions, wt as buildEvalKey, yt as runManifestSchema } from "./runExecution-C24aYsk3.mjs";
+import { C as parseEvalDiscovery, h as persistRunState, r as getTargetEvals$1, t as executeRun } from "./runOrchestration-9XKoYcP9.mjs";
 import { z } from "zod";
 import { readFile } from "node:fs/promises";
 import { relative } from "node:path";

package/dist/{runExecution-D-CnSRYy.mjs → runExecution-C24aYsk3.mjs} RENAMED Viewed

@@ -991,7 +991,12 @@ function getCaseRowCaseKey(row) {
 //#region ../shared/src/schemas/config.ts
 /** Strategy used to collapse repeated trials into one stored case result. */
 const trialSelectionModeSchema = z.enum(["lowestScore", "median"]);
-/** Built-in eval-level output/column keys. */
+/**
+* Built-in eval-level output/column keys.
+*
+* `costUsd` controls the default LLM cost family: actual billed cost plus the
+* normalized `costUsdWithoutCache` and `costUsdWarmedCache` chart outputs.
+*/
 const defaultConfigKeySchema = z.enum([
 	"apiCalls",
 	"costUsd",
@@ -6301,21 +6306,40 @@ const costNumberFormat = {
 	prefix: "$",
 	maxDecimalPlaces: 4
 };
-const DEFAULT_COLUMNS = {
-	apiCalls: {
-		label: "API Calls",
+const DEFAULT_COST_COLUMNS = {
+	costUsd: {
+		label: "Cost",
 		format: "number",
-		numberFormat: countNumberFormat,
+		numberFormat: costNumberFormat,
 		align: "right",
 		hideIfNoValue: true
 	},
-	costUsd: {
-		label: "Cost",
+	costUsdWithoutCache: {
+		label: "Cost Without Cache",
 		format: "number",
 		numberFormat: costNumberFormat,
 		align: "right",
+		hideInTable: true,
 		hideIfNoValue: true
 	},
+	costUsdWarmedCache: {
+		label: "Cost Warmed Cache",
+		format: "number",
+		numberFormat: costNumberFormat,
+		align: "right",
+		hideInTable: true,
+		hideIfNoValue: true
+	}
+};
+const DEFAULT_COLUMNS = {
+	apiCalls: {
+		label: "API Calls",
+		format: "number",
+		numberFormat: countNumberFormat,
+		align: "right",
+		hideIfNoValue: true
+	},
+	costUsd: { ...DEFAULT_COST_COLUMNS.costUsd },
 	llmTurns: {
 		label: "LLM Turns",
 		format: "number",
@@ -6389,8 +6413,16 @@ function mergeDefaultColumns(params) {
 		};
 		return Object.keys(merged).length > 0 ? merged : void 0;
 	}
+	const defaults = {};
+	for (const key of activeKeys) {
+		defaults[key] = DEFAULT_COLUMNS[key];
+		if (key === "costUsd") {
+			defaults.costUsdWithoutCache = DEFAULT_COST_COLUMNS.costUsdWithoutCache;
+			defaults.costUsdWarmedCache = DEFAULT_COST_COLUMNS.costUsdWarmedCache;
+		}
+	}
 	return {
-		...Object.fromEntries(activeKeys.map((key) => [key, DEFAULT_COLUMNS[key]])),
+		...defaults,
 		...params.globalColumns,
 		...params.columns
 	};
@@ -6445,13 +6477,29 @@ function appendDefaultCharts(params) {
 		hideIfNoValue: true,
 		dedupeConsecutiveValues: true,
 		type: "area",
-		metrics: [{
-			source: "column",
-			key: "costUsd",
-			aggregate: "avg",
-			label: "Cost",
-			color: "warning"
-		}]
+		metrics: [
+			{
+				source: "column",
+				key: "costUsd",
+				aggregate: "avg",
+				label: "Actual",
+				color: "warning"
+			},
+			{
+				source: "column",
+				key: "costUsdWithoutCache",
+				aggregate: "avg",
+				label: "Without Cache",
+				color: "error"
+			},
+			{
+				source: "column",
+				key: "costUsdWarmedCache",
+				aggregate: "avg",
+				label: "Warmed Cache",
+				color: "success"
+			}
+		]
 	});
 	const inputTokenMetrics = [
 		activeKeys.has("inputTokens") ? {
@@ -6538,10 +6586,24 @@ function getMaxLlmTurns(calls) {
 }
 function assignIfMissing(params) {
 	if (!params.activeKeys.has(params.key)) return;
+	assignOutputIfMissing({
+		outputs: params.outputs,
+		key: params.key,
+		value: params.value
+	});
+}
+function assignOutputIfMissing(params) {
 	if (params.key in params.outputs) return;
 	if (params.value === void 0) return;
 	params.outputs[params.key] = params.value;
 }
+function sumSimulatedCost(params) {
+	return sumNullable(params.calls.map((call) => simulateLlmCallCost({
+		entry: call,
+		pricing: params.llmCallsConfig.pricing,
+		scenario: params.scenario
+	}).totalCostUsd));
+}
 function addDefaultOutputs(params) {
 	const activeKeys = new Set(getActiveDefaultConfigKeys(params));
 	if (activeKeys.size === 0) return;
@@ -6566,6 +6628,26 @@ function addDefaultOutputs(params) {
 		value: sumNullable(calls.map((call) => call.costUsd)),
 		activeKeys
 	});
+	if (activeKeys.has("costUsd")) {
+		assignOutputIfMissing({
+			outputs: params.outputs,
+			key: "costUsdWithoutCache",
+			value: sumSimulatedCost({
+				calls,
+				llmCallsConfig: params.llmCallsConfig,
+				scenario: "noCache"
+			})
+		});
+		assignOutputIfMissing({
+			outputs: params.outputs,
+			key: "costUsdWarmedCache",
+			value: sumSimulatedCost({
+				calls,
+				llmCallsConfig: params.llmCallsConfig,
+				scenario: "withBaseCaching"
+			})
+		});
+	}
 	assignIfMissing({
 		outputs: params.outputs,
 		key: "inputTokens",

package/dist/{runOrchestration-Basvyp4u.mjs → runOrchestration-9XKoYcP9.mjs} RENAMED Viewed

@@ -1,4 +1,4 @@
-import { Ct as buildCaseKey, Dt as caseRowSchema, Et as caseDetailSchema, Ft as runWithEvalRegistry, Tt as getCaseRowCaseKey, X as runWithEvalClock, _t as validateEvalTagName, bt as runSummarySchema, d as loadEvalModule, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as commitPendingCacheWrites, ht as dedupeEvalTags, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromChildStatuses, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveStatusFromCaseRows, q as runInEvalRuntimeScope, t as filterEvalCases, u as runWithModuleIsolation, vt as validateTagsFilterExpression, yt as runManifestSchema } from "./runExecution-D-CnSRYy.mjs";
+import { Ct as buildCaseKey, Dt as caseRowSchema, Et as caseDetailSchema, Ft as runWithEvalRegistry, Tt as getCaseRowCaseKey, X as runWithEvalClock, _t as validateEvalTagName, bt as runSummarySchema, d as loadEvalModule, f as resolveEvalDefaultConfig, ft as deriveScopedSummaryFromCases, gt as matchesTagsFilter, h as commitPendingCacheWrites, ht as dedupeEvalTags, i as isCaseChildMessage, m as buildDeclaredColumnDefs, mt as deriveStatusFromChildStatuses, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, pt as deriveStatusFromCaseRows, q as runInEvalRuntimeScope, t as filterEvalCases, u as runWithModuleIsolation, vt as validateTagsFilterExpression, yt as runManifestSchema } from "./runExecution-C24aYsk3.mjs";
 import { Result, resultify } from "t-result";
 import { readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";

package/dist/{runner-B6UT1K7L.mjs → runner-5eU-FLHV.mjs} RENAMED Viewed

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-DwNb5TCb.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-BEQGkHF0.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-DwNb5TCb.mjs → runner-BEQGkHF0.mjs} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-CPBIcMP-.mjs";
-import "./src-SixIk0b7.mjs";
+import { n as createRunner } from "./cli-CCHcjbC1.mjs";
+import "./src-C8n7QANC.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance({ loadEnv = true } = {}) {

package/dist/{src-SixIk0b7.mjs → src-C8n7QANC.mjs} RENAMED Viewed

@@ -1,5 +1,5 @@
-import { Nt as defineEval$1, W as matchesEvalTags$1 } from "./runExecution-D-CnSRYy.mjs";
-import "./cli-CPBIcMP-.mjs";
+import { Nt as defineEval$1, W as matchesEvalTags$1 } from "./runExecution-C24aYsk3.mjs";
+import "./cli-CCHcjbC1.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.61.2",
+  "version": "0.62.0",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -33,8 +33,8 @@
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/sdk": "0.0.1",
-    "@agent-evals/shared": "0.0.1"
+    "@agent-evals/shared": "0.0.1",
+    "@agent-evals/sdk": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",

package/skills/agent-eval/SKILL.md CHANGED Viewed

@@ -109,7 +109,7 @@ export async function runRefundWorkflow(input: RefundInput) {
 }
 ```
-Span `kind` values are open-ended strings. Use familiar kinds such as `agent`, `tool`, `llm`, `api`, `retrieval`, `scorer`, or `checkpoint` when they fit, and preserve external tracer kinds such as `mastra.workflow.step` when they are more specific. Only the `input` and `output` span attributes are promoted automatically in the trace tree; use `traceDisplay` for other span attributes such as `model` or `usage`. Eval-level LLM usage outputs, columns, stats, and charts are derived from matching LLM spans by default. Prefer `llmCalls.pricing` for LLM-call cost display; built-in costs ignore span `costUsd` attributes.
+Span `kind` values are open-ended strings. Use familiar kinds such as `agent`, `tool`, `llm`, `api`, `retrieval`, `scorer`, or `checkpoint` when they fit, and preserve external tracer kinds such as `mastra.workflow.step` when they are more specific. Only the `input` and `output` span attributes are promoted automatically in the trace tree; use `traceDisplay` for other span attributes such as `model` or `usage`. Eval-level LLM usage outputs, columns, stats, and charts are derived from matching LLM spans by default. Prefer `llmCalls.pricing` for LLM-call cost display; built-in costs ignore span `costUsd` attributes and derive normalized cost outputs for no-cache and warmed-cache chart comparisons.
 Use `captureEvalSpanError(error)` for recoverable errors on the active `evalTracer.span(...)`, such as optional model/tool failures that fall back and continue. You can pass one error, multiple error arguments, or an array. The span is still marked `error`. Pass `'warning'` or `{ level: 'warning' }` as the final argument for diagnostics that should not change an otherwise successful span's status.
@@ -212,7 +212,7 @@ See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape (forma
 - `tracingAssertions` is a single function that can be authored globally or locally on one eval when a finished-trace invariant should pass or fail the case without creating a fake score column. It receives the same `{ trace, input, case }` context as `deriveFromTracing`; call `evalAssert(...)` or `evalExpect(...)` inside it. Useful trace helpers include `trace.findSpan(name)`, `trace.findSpans(name)`, `trace.hasSpan(name)`, `trace.findSpansByKind(kind)`, `trace.findToolCallSpans()`, `trace.listToolCallSpanNames()`, `trace.hasToolCallSpan(name)`, `trace.getToolCallSpans(name)`, `trace.getToolCallSpanCount(toolName)`, `trace.hasToolCallSpanCount(toolName, expectedCalls)`, `trace.listSpanNames(kind?)`, `trace.listSpanNamesDfs(kind?)`, and `trace.flattenDfs()`. The tool-call helpers include both `kind: 'tool'` spans and imported execution spans recorded as `kind: 'tool_call'`. Tool-name checks and counts match the span `name` as well as GenAI/Mastra identity attributes such as `genAI["gen_ai.tool.name"]` and `mastra.entityName`; list helpers prefer those tool identity attributes when present. `getToolCallSpans(name)` returns one normalized object per matching call, including parsed `arguments`, parsed `result`, `description`, `toolType`, `attributes`, and the original `span`.
 - `traceDisplay` promotes selected span attributes into the trace tree and detail pane; it supports aggregation across subtrees (`scope`, `mode`) and user-defined `transform(...)` for derived views (e.g. currency conversion). See the `TraceDisplayInputConfig` type.
 - `llmCalls` (in `agent-evals.config.ts`) configures how LLM-call spans are summarized for review. Defaults to `kind: 'llm'` spans with `model`, `usage.*`, `latencyMs`, `input`, `output`, etc. read from conventional attribute paths. The default `steps` path reads an array from `span.attributes.steps`; if it is missing, direct child `model_step` spans are shown as that call's steps. Tool calls are aggregated from the configured `toolCalls` path plus step-level `toolCalls` on authored step arrays or direct `model_step` child spans, including Mastra's serialized `mastra.model_step.output` format, and child `tool_call` execution spans under each model step. `latencyMs` is time to first token; duration, total tokens, output tokens/sec, and USD costs are derived. Override `kinds` to broaden the filter, override `attributes.<field>` for non-default primitive span shapes, configure model-keyed `pricing` to derive USD costs from token counts, with nested `providers` entries for provider-specific rates, add `costCurrencies` to show converted cost columns in the expanded breakdown table only, add `derivedAttributes` to persist computed values back onto matching LLM spans before trace consumers run, and add entries to `metrics` to surface arbitrary user metrics (`format: 'string' | 'number' | 'duration' | 'json' | 'boolean'`, `placements: ['header' | 'body']`). `derivedAttributes` can be a keyed map for one-off fields or one callback that returns multiple path/value pairs. Derived keys are dot-paths under `span.attributes`; return `undefined` to skip one span or one returned key.
-- Default usage config derives missing eval outputs from matching LLM/API spans before `outputsSchema` and scores run: `apiCalls`, `costUsd`, `llmTurns`, `inputTokens`, `outputTokens`, `totalTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `reasoningTokens`, and `llmDurationMs`. Authored outputs and column overrides win. The web UI fills in baseline run-health stats (`cases`, `passRate`, `duration`) and a pass-rate/duration history chart when an eval has not already authored equivalent run-health UI. If discovery metadata is missing but saved runs contain runtime columns such as `costUsd`, `inputTokens`, or `apiCalls`, the single-eval page can infer the standard usage stats and charts from those saved run values. Default usage columns, stats, and charts use `hideIfNoValue: true`. Default LLM usage charts configure cost, input tokens, and output tokens separately and use `dedupeConsecutiveValues: true` to skip repeated adjacent chart values. `totalTokens` is input + output only; cache read/write tokens stay separate and affect `costUsd` at their own rates. `llmTurns` is the maximum per-call turn count in the case run, using configured steps when available and otherwise one turn per matched LLM call span. Derived base input cost uses `inputTokens - cachedInputTokens - cacheCreationInputTokens` so cache details are not double-counted. `cacheCreationInputTokens` is the total cache-write count; optional `cacheCreationInput1hTokens` only splits that total for 1-hour write pricing via `cacheCreationInput1hUsdPerMillion`. `llmDurationMs` sums elapsed matched LLM span durations; it is not time-to-first-token latency. Remove defaults globally or per eval with `removeDefaultConfig: true` or a key list such as `removeDefaultConfig: ['apiCalls', 'reasoningTokens']`.
+- Default usage config derives missing eval outputs from matching LLM/API spans before `outputsSchema` and scores run: `apiCalls`, `costUsd`, `costUsdWithoutCache`, `costUsdWarmedCache`, `llmTurns`, `inputTokens`, `outputTokens`, `totalTokens`, `cachedInputTokens`, `cacheCreationInputTokens`, `reasoningTokens`, and `llmDurationMs`. Authored outputs and column overrides win. The web UI fills in baseline run-health stats (`cases`, `passRate`, `duration`) and a pass-rate/duration history chart when an eval has not already authored equivalent run-health UI. If discovery metadata is missing but saved runs contain runtime columns such as `costUsd`, `inputTokens`, or `apiCalls`, the single-eval page can infer the standard usage stats and charts from those saved run values. Default usage columns, stats, and charts use `hideIfNoValue: true`. Default LLM usage charts configure cost, input tokens, and output tokens separately and use `dedupeConsecutiveValues: true` to skip repeated adjacent chart values. The default LLM cost chart compares actual billed cost, no-cache normalized cost, and warmed-cache normalized cost. `totalTokens` is input + output only; cache read/write tokens stay separate and affect `costUsd` at their own rates. `costUsdWithoutCache` bills all input tokens at the base input rate; `costUsdWarmedCache` treats cacheable tokens as warmed cache reads. `llmTurns` is the maximum per-call turn count in the case run, using configured steps when available and otherwise one turn per matched LLM call span. Derived base input cost uses `inputTokens - cachedInputTokens - cacheCreationInputTokens` so cache details are not double-counted. `cacheCreationInputTokens` is the total cache-write count; optional `cacheCreationInput1hTokens` only splits that total for 1-hour write pricing via `cacheCreationInput1hUsdPerMillion`. `llmDurationMs` sums elapsed matched LLM span durations; it is not time-to-first-token latency. Remove defaults globally or per eval with `removeDefaultConfig: true` or a key list such as `removeDefaultConfig: ['apiCalls', 'reasoningTokens']`; removing `costUsd` removes the default cost family.
 - `apiCalls` (in `agent-evals.config.ts`) configures how API-call spans are summarized for review. Defaults to `kind: 'api'`, `'http'`, `'http.client'`, and `'fetch'` spans with `method`, `url`, `statusCode`, `request`, `routeAlias`, `response`, `requestBody`, `responseBody`, `headers`, `durationMs`, and `error` read from conventional attribute paths. Override `kinds` or `attributes.<field>` for external tracers. Set a per-span `routeAlias` attribute such as `/v3/tabs/:id` to group dynamic URL paths in API-call route labels and endpoint charts while preserving original URLs in row details. Add `derivedAttributes` as a keyed map or object-returning callback for computed persisted API span attributes, and add `metrics` with the same formats and placements as LLM-call metrics.
 - `runLogs` (in `agent-evals.config.ts`) controls case log capture. Use `runLogs: { captureConsole: false }` to keep console output in the terminal without persisting console calls to case details. Manual `evalLog(...)` calls are still captured. Captured log locations store the selected user-facing source frame and the full JavaScript stack so agents can inspect additional frames in persisted artifacts when diagnosing where a log came from.