@tangle-network/agent-eval 0.186.0 → 0.186.2

package/dist/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { i as runRolloutReleaseCli } from "./hf-dataset-CRqtSjnR.js";
 import { t as createChatClient } from "./chat-client-DqPqWzVV.js";
-import { t as runAnalystBenchmarkCommand } from "./benchmark-command-tBgs-_AM.js";
+import { t as runAnalystBenchmarkCommand } from "./benchmark-command-NATyhXB4.js";
 import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-DmpcFiVK.js";
 import { t as runSupervisorRunCommand } from "./report-command-BvqaxlVG.js";
 import { writeFileSync } from "node:fs";

package/dist/diagnosis/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/diagnosis/findings.ts","../../src/diagnosis/secret-filter.ts","../../src/diagnosis/spans.ts","../../src/diagnosis/deterministic.ts","../../src/diagnosis/model-pass.ts","../../src/diagnosis/engine.ts"],"mappings":";;;;;;;;;;;;;cASa;KAGD;;KAGA;UAEK;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;EACV;EACA;EACA,UAAU;;EAEV;EACA;EACA,YAAY;EACZ;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,QAAQ;EACR;;UAGe;EACf;EACA;IACE;IACA;IACA;IACA;;EAEF;IACE,cAAc,eAAe;IAC7B,SAAS;IACT,YAAY;;EAEd,UAAU;;;;;;;;iBAiBI,0BACd,mBACA;EAAW,UAAU;;;;;;;;;;;;;;;;;;;;;;;UClEN;EACf;EACA,SAAS;;EAET,UAAU,kBAAkB;;UAGb;EACf;EACA,QAAQ;;;;;;;cAWG,2BAAyB;cAGzB,iCAAiC;;iBA8D9B,cACd,eACA,SAAQ,oBACR,iBAAgB;;iBAwBF,YACd,eACA,iBAAgB;;iBAWF,kBAAkB,gBAAgB,QAAQ;;;KC1H9C;UAEK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA,QAAQ;EACR;EACA;EACA;;;;;;;EAOA;EACA,YAAY;;UAGG;EACf;EACA;;EAEA;;EAEA;;EAEA;EACA,SAAS;;EAET;;iBAqBc,mBAAmB;;;;;;;iBAUnB,YACd,yBACA;EAAW;;EACR,OAAO;EAAiB,QAAQ;;;;;;;iBA+GrB,YAAY;;;UCtJX;EACf;EACA;EACA,aAAa;EACb;EACA;EACA;;EAEA,WAAW;EACX;;UAGe;EACf,YAAY;EACZ,cAAc,eAAe;EAC7B,SAAS;EACT,OAAO;EACP,UAAU;EACV;;;;UC/Ce;;EAEf,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IAAY;IAA4B;;;UAGzB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD,OAAO;EACP;;;;UC/Be;;EAEf;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;IAAa,MAAM;MAAQ;QAAe;MAA6B;;KAG7D;EACN;EAAuB;EAAiB,SAAS;;EACjD;EAAe,OAAO;EAAuB,SAAS;;UAE3C;EACf;EACA;EACA;EACA;;UAGe;;EAEf,UAAU;;EAEV,WAAW;;EAEX,OAAO;;EAEP,WAAW;IAAQ;IAAiB;;EACpC,OAAO;EACP,aAAa;EACb,QAAQ;;EAER,UAAU;EACV;IACE;IACA;IACA,MAAM;IACN;IACA,OAAO;;EAET;IAAU;IAA8B;;;cAG7B;iBAIS,cACpB,2BACA,SAAS,kBACT,SAAS,mBACR,QAAQ"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/diagnosis/findings.ts","../../src/diagnosis/secret-filter.ts","../../src/diagnosis/spans.ts","../../src/diagnosis/deterministic.ts","../../src/diagnosis/model-pass.ts","../../src/diagnosis/engine.ts"],"mappings":";;;;;;;;;;;;;cASa;KAGD;;KAGA;UAEK;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;EACV;EACA;EACA,UAAU;;EAEV;EACA;EACA,YAAY;EACZ;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,QAAQ;EACR;;UAGe;EACf;EACA;IACE;IACA;IACA;IACA;;EAEF;IACE,cAAc,eAAe;IAC7B,SAAS;IACT,YAAY;;EAEd,UAAU;;;;;;;;iBAiBI,0BACd,mBACA;EAAW,UAAU;;;;;;;;;;;;;;;;;;;;;;;UClEN;EACf;EACA,SAAS;;EAET,UAAU,kBAAkB;;UAGb;EACf;EACA,QAAQ;;;;;;;cAWG,2BAAyB;cAGzB,iCAAiC;;iBA8D9B,cACd,eACA,SAAQ,oBACR,iBAAgB;;iBAwBF,YACd,eACA,iBAAgB;;iBAWF,kBAAkB,gBAAgB,QAAQ;;;KC1H9C;UAEK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA,QAAQ;EACR;EACA;EACA;;;;;;;EAOA;EACA,YAAY;;UAGG;EACf;EACA;;EAEA;;EAEA;;EAEA;EACA,SAAS;;EAET;;iBA0Bc,mBAAmB;;;;;;;iBAUnB,YACd,yBACA;EAAW;;EACR,OAAO;EAAiB,QAAQ;;;;;;;iBA+GrB,YAAY;;;UC3JX;EACf;EACA;EACA,aAAa;EACb;EACA;EACA;;EAEA,WAAW;EACX;;UAGe;EACf,YAAY;EACZ,cAAc,eAAe;EAC7B,SAAS;EACT,OAAO;EACP,UAAU;EACV;;;;UC/Ce;;EAEf,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IAAY;IAA4B;;;UAGzB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD,OAAO;EACP;;;;UC/Be;;EAEf;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;IAAa,MAAM;MAAQ;QAAe;MAA6B;;KAG7D;EACN;EAAuB;EAAiB,SAAS;;EACjD;EAAe,OAAO;EAAuB,SAAS;;UAE3C;EACf;EACA;EACA;EACA;;UAGe;;EAEf,UAAU;;EAEV,WAAW;;EAEX,OAAO;;EAEP,WAAW;IAAQ;IAAiB;;EACpC,OAAO;EACP,aAAa;EACb,QAAQ;;EAER,UAAU;EACV;IACE;IACA;IACA,MAAM;IACN;IACA,OAAO;;EAET;IAAU;IAA8B;;;cAG7B;iBAIS,cACpB,2BACA,SAAS,kBACT,SAAS,mBACR,QAAQ"}

package/dist/diagnosis/index.js

@@ -151,18 +151,22 @@ function countGroups(pattern) {
 * not included they are removed before anything reads the span, rather than
 * trusting pattern redaction over free-form prose.
 */
-const CONTENT_ATTRIBUTE = /^(?:input\.value|output\.value|input|output|result|prompt|completion|text|thinking|message|messages|tool\.(?:input|output|arguments|result)|tool_input|tool_output|arguments|gen_ai\.(?:prompt|completion|input\.messages|output\.messages|system_instructions|tool\.call\.(?:arguments|result))(?:\..*)?|llm\.(?:input|output)_messages(?:\..*)?|llm\.prompts?(?:\..*)?|content|body|request|response|command)$|\.content$/;
+const CONTENT_ATTRIBUTE = /^(?:input\.value|output\.value|input|output|result|prompt|completion|text|thinking|message|messages|tool\.(?:input|output|arguments|result)|tool_input|tool_output|tool_arguments|arguments|args|full_command|gen_ai\.(?:prompt|completion|input\.messages|output\.messages|system_instructions|tool\.call\.(?:arguments|result))(?:\..*)?|llm\.(?:input|output)_messages(?:\..*)?|llm\.prompts?(?:\..*)?|content|body|request|response|command)$|\.content$/i;
+const PROSE_ATTRIBUTE = /(?:^|[._])(?:error|exception|status)(?:[._][a-zA-Z0-9]+)*[._](?:message|stacktrace|stack|description|details|reason)$|^(?:error|exception|status)$|^(?:log|event)[._]message$|^otel\.status_description$|^(?:events?|logs?)$/i;
 const INPUT_ATTRIBUTE_KEYS = [
 	"input.value",
 	"input",
 	"tool.input",
 	"tool.arguments",
 	"tool_input",
+	"tool_arguments",
 	"arguments",
+	"args",
+	"full_command",
 	"gen_ai.tool.call.arguments"
 ];
 function isContentAttribute(key) {
-	return CONTENT_ATTRIBUTE.test(key);
+	return CONTENT_ATTRIBUTE.test(key) || PROSE_ATTRIBUTE.test(key);
 }
 /**
 * Filter secrets from every string, drop content attributes unless content is
@@ -208,7 +212,7 @@ function ingestSpans(raw, options) {
 		const inputDigest = digestInput(merged, digestKey);
 		const attributes = redactSecretsDeep(merged, secrets);
 		if (!options.contentIncluded) {
-			for (const key of Object.keys(attributes)) if (isContentAttribute(key)) {
+			for (const [key, value] of Object.entries(attributes)) if (isContentAttribute(key) || value !== null && typeof value === "object") {
 				delete attributes[key];
 				dropped.add(key);
 			}
@@ -227,7 +231,7 @@ function ingestSpans(raw, options) {
 			startMs: epochMillis(record.start_time ?? record.startTime ?? record.startTimeUnixNano),
 			endMs: epochMillis(record.end_time ?? record.endTime ?? record.endTimeUnixNano),
 			status: status.code,
-			statusMessage: options.contentIncluded ? statusMessage : truncate(statusMessage, 160),
+			statusMessage: options.contentIncluded ? statusMessage : null,
 			toolName,
 			model,
 			inputDigest,
@@ -355,10 +359,6 @@ function firstString(attributes, keys) {
 function stringOr(value, fallback) {
 	return typeof value === "string" && value.length > 0 ? value : fallback;
 }
-function truncate(value, max) {
-	if (value === null || value.length <= max) return value;
-	return `${value.slice(0, max)}…`;
-}
 /** The analysis each capability gates, as the report names it. */
 const CAPABILITY_ANALYSES = {
 	"token-accounting": "token accounting",
@@ -742,6 +742,7 @@ function modelContractLines(subject) {
 		...subject === "internal" ? ["An \"operator\" row: {\"kind\":\"operator\",\"claim\":\"one sentence critiquing how the human operator instructed or steered the agent\",\"evidence\":[\"<span_id>\", ...]}."] : [],
 		"Add \"project\": \"<repo or project name>\" to any row when the run shows which project it belongs to.",
 		"Every \"finding\" and \"operator\" row cites 1-8 span ids copied VERBATIM from the \"id\" fields of the trajectory. Rows citing ids that are not in the trajectory are discarded.",
+		"For elapsed wall time, compare the start and end timestamps and count overlapping spans only once. Do not add overlapping ms durations or call their sum a stall. If timestamps are missing, do not claim a total wall delay.",
 		"Keep every string under 400 characters. Report only what the trajectory shows; an empty rows array is a valid answer."
 	];
 }
@@ -1103,6 +1104,10 @@ function renderSpan(span, attributeChars, contentIncluded) {
 	if (span.statusMessage) out.status_message = clip(span.statusMessage, attributeChars);
 	const duration = durationMs(span);
 	if (duration !== null) out.ms = duration;
+	if (span.startMs !== null && span.endMs !== null) {
+		out.start = new Date(span.startMs).toISOString();
+		out.end = new Date(span.endMs).toISOString();
+	}
 	const attrs = {};
 	for (const [key, value] of Object.entries(span.attributes)) {
 		if (SKIP_ATTRIBUTES.has(key)) continue;