@ls-stack/agent-eval 0.58.5 → 0.59.1

package/dist/{runExecution-BMnJXWhN.mjs → runExecution-C3XVZHRC.mjs}

@@ -5142,8 +5142,88 @@ function buildTraceTree(spans, checkpoints) {
 		visit(null);
 		return result;
 	};
+	const isRecord = (value) => {
+		return typeof value === "object" && value !== null;
+	};
+	const readRecordValue = (value, key) => {
+		if (!isRecord(value)) return void 0;
+		const child = value[key];
+		return isRecord(child) ? child : void 0;
+	};
+	const readStringValue = (value, key) => {
+		if (!isRecord(value)) return void 0;
+		const child = value[key];
+		return typeof child === "string" && child.length > 0 ? child : void 0;
+	};
+	const readValue = (value, key) => {
+		if (!isRecord(value)) return void 0;
+		return value[key];
+	};
+	const parseMaybeJson = (value) => {
+		if (typeof value !== "string") return value;
+		const parsed = resultify(() => JSON.parse(value));
+		return parsed.error ? value : parsed.value;
+	};
+	const firstDefined = (values) => {
+		return values.find((value) => value !== void 0);
+	};
+	const getToolCallMetadata = (span) => {
+		const attributes = span.attributes;
+		return {
+			attributes,
+			genAI: readRecordValue(attributes, "genAI"),
+			mastra: readRecordValue(attributes, "mastra"),
+			toolAttributes: readRecordValue(attributes, "attributes")
+		};
+	};
 	const isToolCallSpan = (span) => {
-		return span.kind === "tool" || span.kind === "tool_call";
+		const { attributes, genAI, mastra } = getToolCallMetadata(span);
+		return span.kind === "tool" || span.kind === "tool_call" || readStringValue(attributes, "gen_ai.tool.type") === "tool" || readStringValue(genAI, "gen_ai.tool.type") === "tool" || readStringValue(genAI, "mastra.span.type") === "tool_call" || readStringValue(mastra, "type") === "tool_call" || readStringValue(mastra, "entityType") === "tool";
+	};
+	const getToolCallIdentityNames = (span) => {
+		const { attributes, genAI, mastra } = getToolCallMetadata(span);
+		return [
+			readStringValue(attributes, "gen_ai.tool.name"),
+			readStringValue(genAI, "gen_ai.tool.name"),
+			readStringValue(mastra, "entityName"),
+			readStringValue(mastra, "entityId"),
+			span.name
+		].filter((name) => name !== void 0);
+	};
+	const getPreferredToolCallName = (span) => {
+		return getToolCallIdentityNames(span)[0] ?? span.name;
+	};
+	const toolCallSpanMatchesName = (span, toolName) => {
+		return getToolCallIdentityNames(span).includes(toolName);
+	};
+	const countToolCallSpans = (toolName) => {
+		return spans.filter((span) => {
+			return isToolCallSpan(span) && toolCallSpanMatchesName(span, toolName);
+		}).length;
+	};
+	const buildToolCallSpan = (span) => {
+		const { attributes, genAI, toolAttributes } = getToolCallMetadata(span);
+		return {
+			name: getPreferredToolCallName(span),
+			spanName: span.name,
+			kind: span.kind,
+			arguments: parseMaybeJson(firstDefined([
+				readValue(attributes, "gen_ai.tool.call.arguments"),
+				readValue(genAI, "gen_ai.tool.call.arguments"),
+				readValue(attributes, "arguments"),
+				readValue(attributes, "input")
+			])),
+			result: parseMaybeJson(firstDefined([
+				readValue(attributes, "gen_ai.tool.call.result"),
+				readValue(genAI, "gen_ai.tool.call.result"),
+				readValue(attributes, "result"),
+				readValue(attributes, "output")
+			])),
+			description: readStringValue(attributes, "gen_ai.tool.description") ?? readStringValue(genAI, "gen_ai.tool.description") ?? readStringValue(toolAttributes, "toolDescription"),
+			toolType: readStringValue(attributes, "gen_ai.tool.type") ?? readStringValue(genAI, "gen_ai.tool.type") ?? readStringValue(toolAttributes, "toolType"),
+			attributes,
+			span
+		};
 	};
 	const filterSpanNames = (sourceSpans, kind) => {
 		return sourceSpans.filter((span) => kind === void 0 || span.kind === kind).map((span) => span.name);
@@ -5167,15 +5247,23 @@ function buildTraceTree(spans, checkpoints) {
 			return spans.filter(isToolCallSpan);
 		},
 		listToolCallSpanNames() {
-			return spans.filter(isToolCallSpan).map((span) => span.name);
+			return spans.filter(isToolCallSpan).map(getPreferredToolCallName);
 		},
 		hasToolCallSpan(name) {
-			return spans.some((s) => isToolCallSpan(s) && s.name === name);
+			return spans.some((s) => {
+				return isToolCallSpan(s) && toolCallSpanMatchesName(s, name);
+			});
 		},
-		hasNToolCallSpans(toolName, expectedCalls) {
+		getToolCallSpans(name) {
 			return spans.filter((span) => {
-				return isToolCallSpan(span) && span.name === toolName;
-			}).length === expectedCalls;
+				return isToolCallSpan(span) && toolCallSpanMatchesName(span, name);
+			}).map(buildToolCallSpan);
+		},
+		getToolCallSpanCount(toolName) {
+			return countToolCallSpans(toolName);
+		},
+		hasToolCallSpanCount(toolName, expectedCalls) {
+			return countToolCallSpans(toolName) === expectedCalls;
 		},
 		listSpanNames(kind) {
 			return filterSpanNames(spans, kind);

package/dist/{runOrchestration-CvbTAoEb.mjs → runOrchestration-B5An-AEi.mjs}

@@ -1,4 +1,4 @@
-import { Dt as getCaseRowCaseKey, Lt as runWithEvalRegistry, Ot as caseDetailSchema, Q as runWithEvalClock, St as runSummarySchema, Tt as buildCaseKey, Y as runInEvalRuntimeScope, _t as dedupeEvalTags, bt as validateTagsFilterExpression, d as loadEvalModule, f as resolveEvalDefaultConfig, g as commitPendingCacheWrites, gt as deriveStatusFromChildStatuses, ht as deriveStatusFromCaseRows, i as isCaseChildMessage, kt as caseRowSchema, m as buildDeclaredColumnDefs, mt as deriveScopedSummaryFromCases, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, t as filterEvalCases, u as runWithModuleIsolation, vt as matchesTagsFilter, xt as runManifestSchema, yt as validateEvalTagName } from "./runExecution-BMnJXWhN.mjs";
+import { Dt as getCaseRowCaseKey, Lt as runWithEvalRegistry, Ot as caseDetailSchema, Q as runWithEvalClock, St as runSummarySchema, Tt as buildCaseKey, Y as runInEvalRuntimeScope, _t as dedupeEvalTags, bt as validateTagsFilterExpression, d as loadEvalModule, f as resolveEvalDefaultConfig, g as commitPendingCacheWrites, gt as deriveStatusFromChildStatuses, ht as deriveStatusFromCaseRows, i as isCaseChildMessage, kt as caseRowSchema, m as buildDeclaredColumnDefs, mt as deriveScopedSummaryFromCases, n as resolveRunnableEvalCases, o as stripTerminalControlCodes, t as filterEvalCases, u as runWithModuleIsolation, vt as matchesTagsFilter, xt as runManifestSchema, yt as validateEvalTagName } from "./runExecution-C3XVZHRC.mjs";
 import { Result, resultify } from "t-result";
 import { readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";

package/dist/{runner-DJJekv9f.mjs → runner-BJXz_V_V.mjs}

@@ -1,2 +1,2 @@
-import { n as initRunner, t as getRunnerInstance } from "./runner-BfHgVhGS.mjs";
+import { n as initRunner, t as getRunnerInstance } from "./runner-C9J-1fkp.mjs";
 export { getRunnerInstance, initRunner };

package/dist/{runner-BfHgVhGS.mjs → runner-C9J-1fkp.mjs}

@@ -1,5 +1,5 @@
-import { n as createRunner } from "./cli-ClAkjTvo.mjs";
-import "./src-DfzidkYr.mjs";
+import { n as createRunner } from "./cli-Dkp2-rBm.mjs";
+import "./src-8dGXUULC.mjs";
 //#region ../../apps/server/src/runner.ts
 let runnerInstance = null;
 function getRunnerInstance() {

package/dist/{src-DfzidkYr.mjs → src-8dGXUULC.mjs}

@@ -1,5 +1,5 @@
-import { Ft as defineEval$1, K as matchesEvalTags$1 } from "./runExecution-BMnJXWhN.mjs";
-import "./cli-ClAkjTvo.mjs";
+import { Ft as defineEval$1, K as matchesEvalTags$1 } from "./runExecution-C3XVZHRC.mjs";
+import "./cli-Dkp2-rBm.mjs";
 //#region src/index.ts
 /** Register an eval definition with typed tag support. */
 function defineEval(definition) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ls-stack/agent-eval",
-  "version": "0.58.5",
+  "version": "0.59.1",
   "type": "module",
   "bin": {
     "agent-evals": "./dist/bin.mjs"
@@ -32,9 +32,9 @@
   "devDependencies": {
     "@types/node": "^24.7.2",
     "typescript": "^5.9.2",
-    "@agent-evals/sdk": "0.0.1",
     "@agent-evals/runner": "0.0.1",
-    "@agent-evals/shared": "0.0.1"
+    "@agent-evals/shared": "0.0.1",
+    "@agent-evals/sdk": "0.0.1"
   },
   "scripts": {
     "build": "pnpm --filter @agent-evals/web build && pnpm --filter @agent-evals/shared build && pnpm --filter @agent-evals/sdk build && pnpm --filter @agent-evals/runner build && tsdown --filter cli-js && tsdown --filter cli-types",

package/skills/agent-eval/SKILL.md

@@ -375,11 +375,19 @@ See `EvalScoreDef` / `EvalManualScoreDef` in the types for the full shape
   `trace.hasSpan(name)`, `trace.findSpansByKind(kind)`,
   `trace.findToolCallSpans()`, `trace.listToolCallSpanNames()`,
   `trace.hasToolCallSpan(name)`,
-  `trace.hasNToolCallSpans(toolName, expectedCalls)`,
+  `trace.getToolCallSpans(name)`,
+  `trace.getToolCallSpanCount(toolName)`,
+  `trace.hasToolCallSpanCount(toolName, expectedCalls)`,
   `trace.listSpanNames(kind?)`, `trace.listSpanNamesDfs(kind?)`, and
   `trace.flattenDfs()`.
   The tool-call helpers include both `kind: 'tool'` spans and imported
-  execution spans recorded as `kind: 'tool_call'`.
+  execution spans recorded as `kind: 'tool_call'`. Tool-name checks and counts
+  match the span `name` as well as GenAI/Mastra identity attributes such as
+  `genAI["gen_ai.tool.name"]` and `mastra.entityName`; list helpers prefer
+  those tool identity attributes when present. `getToolCallSpans(name)`
+  returns one normalized object per matching call, including parsed
+  `arguments`, parsed `result`, `description`, `toolType`, `attributes`, and
+  the original `span`.
 - `traceDisplay` promotes selected span attributes into the trace tree and
   detail pane; it supports aggregation across subtrees (`scope`, `mode`) and
   user-defined `transform(...)` for derived views (e.g. currency conversion).