@tangle-network/agent-eval 0.216.0 → 0.217.1

package/dist/diagnosis/index.js

@@ -1,12 +1,15 @@
-import { n as summarizeExecution } from "../analyze-runs-3xpby3p2.js";
-import { c as emptyRedactionReport, f as redactText, u as redact } from "../redact-Dk1ZdJEm.js";
+import { r as summarizeExecution } from "../analyze-runs-DqPJaJ4X.js";
+import { a as redact, i as emptyRedactionReport, s as redactText } from "../redact-mtkTiJVk.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
-import { o as classifyFailureReason } from "../failure-taxonomy-CjabTATj.js";
-import { c as primeProtocolSha256, d as runPrimeExchange, n as buildPrimePrompt } from "../prime-protocol-DcAK9pC8.js";
+import { r as classifyFailureReason } from "../failure-taxonomy-Bx5oWqaW.js";
+import { c as runPrimeExchange, n as buildPrimePrompt, o as primeProtocolSha256 } from "../prime-protocol-BFgGrEMP.js";
 import { t as nodeHttpPrimeBridgeTransport } from "../prime-bridge-transport-Z3cpKGoV.js";
-import { t as fromOtelSpans } from "../otel-spans-C8YIJWy1.js";
+import { t as fromOtelSpans } from "../otel-spans-CdN909k1.js";
 import { createHash, createHmac, randomBytes } from "node:crypto";
 import { MODEL_ATTR_KEYS, SPAN_KINDS, TOOL_NAME_ATTR_KEYS, resolveSpanKind, validateTraceSpans } from "@tangle-network/agent-trace-contract";
+//#region src/diagnosis/first-failure.ts
+/** Span ids listed beside a result; the count carries the rest. */
+const MAX_FIRST_FAILURE_IDS = 8;
 const OUTCOME_ATTR = "agent.outcome";
 /** Rank the first failure of one trace. Throws when the spans belong to more than one trace. */
 function rankFirstFailure(spans) {
@@ -73,7 +76,7 @@ function rank(traceId, stage, candidates) {
 		message: first.statusMessage,
 		classification,
 		blame: classification?.blame ?? "unknown",
-		later: later.slice(0, 8).map((span) => span.spanId),
+		later: later.slice(0, MAX_FIRST_FAILURE_IDS).map((span) => span.spanId),
 		laterCount: later.length
 	};
 }
@@ -83,7 +86,7 @@ function ambiguous(traceId, stage, candidates, reason) {
 		status: "ambiguous",
 		traceId,
 		stage,
-		candidates: ids.slice(0, 8),
+		candidates: ids.slice(0, MAX_FIRST_FAILURE_IDS),
 		candidateCount: ids.length,
 		reason
 	};
@@ -311,6 +314,28 @@ function firstString(attributes, keys) {
 function stringOr(value, fallback) {
 	return typeof value === "string" && value.length > 0 ? value : fallback;
 }
+//#endregion
+//#region src/diagnosis/deterministic.ts
+/**
+* The deterministic pass: everything the engine can say without a model.
+*
+* It produces the capability table (the trace contract's own validator, so the
+* reasons are the trace's reasons), execution facts with token and cost
+* accounting (agent-eval's `fromOtelSpans` + `summarizeExecution`), and
+* `observed` findings computed directly from span data. Every observed finding
+* carries a measure with its denominator and cites the spans it counted.
+*/
+/** Evidence ids cited per finding; the measure carries the full count. */
+const MAX_EVIDENCE_PER_FINDING = 8;
+/**
+* Identical tool calls with no different tool call between them that count as
+* a repeated-call run. Model turns between the calls do not break the run: an
+* agent that calls, reads, and calls again with the same input is the waste
+* this detects.
+*/
+const REPEAT_RUN_LENGTH = 3;
+/** A tool is reported only once it has at least this many calls. */
+const MIN_TOOL_CALLS = 3;
 /** The analysis each capability gates, as the report names it. */
 const CAPABILITY_ANALYSES = {
 	"token-accounting": "token accounting",
@@ -397,7 +422,7 @@ function toolFailureFindings(tools) {
 	const findings = [];
 	for (const [tool, calls] of byTool) {
 		const failed = calls.filter((span) => span.status === "ERROR");
-		if (calls.length < 3 || failed.length < 2) continue;
+		if (calls.length < MIN_TOOL_CALLS || failed.length < 2) continue;
 		const rate = failed.length / calls.length;
 		const severity = rate >= .5 && failed.length >= 5 ? "high" : rate >= .2 ? "medium" : "low";
 		findings.push({
@@ -411,7 +436,7 @@ function toolFailureFindings(tools) {
 				unit: "%",
 				denominator: `${calls.length} calls to ${tool}`
 			},
-			evidence: failed.slice(0, 8).map((span) => span.spanId),
+			evidence: failed.slice(0, MAX_EVIDENCE_PER_FINDING).map((span) => span.spanId),
 			confidence: "observed",
 			recommendation: `Read the failed ${tool} calls' status messages and fix the most common cause, or give the agent the precondition it keeps missing.`
 		});
@@ -429,22 +454,22 @@ function repeatedCallFindings(traces) {
 			const last = current[current.length - 1];
 			if (last && last.inputDigest === span.inputDigest && (last.toolName ?? last.name) === (span.toolName ?? span.name)) current.push(span);
 			else {
-				if (current.length >= 3) runs.push(current);
+				if (current.length >= REPEAT_RUN_LENGTH) runs.push(current);
 				current = [span];
 			}
 		}
-		if (current.length >= 3) runs.push(current);
+		if (current.length >= REPEAT_RUN_LENGTH) runs.push(current);
 	}
 	if (runs.length === 0) return [];
 	const wasted = runs.reduce((sum, run) => sum + run.length - 1, 0);
 	const tracesAffected = new Set(runs.map((run) => run[0].traceId)).size;
 	runs.sort((a, b) => b.length - a.length);
-	const evidence = runs.flatMap((run) => run.slice(0, 2).map((span) => span.spanId)).slice(0, 8);
+	const evidence = runs.flatMap((run) => run.slice(0, 2).map((span) => span.spanId)).slice(0, MAX_EVIDENCE_PER_FINDING);
 	const rate = wasted / digestCalls;
 	return [{
 		id: stableId("repeated-calls", String(runs.length)),
 		severity: rate >= .1 ? "high" : rate >= .03 ? "medium" : "low",
-		claim: `The agent made the same tool call with the same input 3 or more times with no other tool call in between, ${runs.length} times across ${tracesAffected} runs; the longest streak was ${runs[0].length} calls.`,
+		claim: `The agent made the same tool call with the same input ${REPEAT_RUN_LENGTH} or more times with no other tool call in between, ${runs.length} times across ${tracesAffected} runs; the longest streak was ${runs[0].length} calls.`,
 		consequence: `${wasted} calls repeated the previous tool call with the same input, which is work the agent paid for twice without new information.`,
 		measure: {
 			name: "repeated identical tool calls",
@@ -477,7 +502,7 @@ function failedRunFindings(traces) {
 			unit: "%",
 			denominator: `${traces.size} runs`
 		},
-		evidence: failedRoots.slice(0, 8).map((span) => span.spanId),
+		evidence: failedRoots.slice(0, MAX_EVIDENCE_PER_FINDING).map((span) => span.spanId),
 		confidence: "observed",
 		recommendation: "Group the failed runs by their root status message and fix the largest group first."
 	}];
@@ -556,15 +581,6 @@ function round(value) {
 }
 //#endregion
 //#region src/diagnosis/findings.ts
-/**
-* The diagnosis findings document: what the engine returns and what the
-* diagnosis report renders. It mirrors the kit's JSON Schema
-* (`https://tangle.tools/schemas/diagnosis-findings-v1.json`, owned by the
-* traces repository's diagnosis kit) field for field, including its closed
-* objects, so a document that passes {@link validateDiagnosisFindings} passes
-* that schema.
-*/
-const DIAGNOSIS_FINDINGS_SCHEMA_ID = "https://tangle.tools/schemas/diagnosis-findings-v1.json";
 const SEVERITIES$1 = /* @__PURE__ */ new Set([
 	"critical",
 	"high",
@@ -1102,6 +1118,29 @@ function shortString(value) {
 	if (!trimmed) return void 0;
 	return trimmed.length <= MAX_STRING ? trimmed : `${trimmed.slice(0, MAX_STRING)}…`;
 }
+//#endregion
+//#region src/diagnosis/engine.ts
+/**
+* The agent-failure diagnosis engine: flat spans plus context in, a findings
+* document out.
+*
+* It is a plain function. Its only side effect is the model call made through
+* the caller's transport, and in deterministic mode it makes none. It keeps no
+* storage, files no issues, and retains nothing: where the spans came from and
+* where the findings go belong to the caller. That is what lets one engine
+* serve both the internal daily mining run and a customer diagnosis while their
+* data paths stay separate.
+*
+* Order of operations, each step reading only the previous step's output:
+*   1. secret filter and content drop over every input (spans, focus, topology);
+*   2. the deterministic pass: capability table, execution facts, observed findings;
+*   3. in model mode, the prime-protocol pass over the highest-signal runs;
+*   4. evidence resolution: every cited id must be a span id of the input that
+*      no other run shares; findings left without evidence are rejected.
+*/
+const DEFAULT_MAX_MODEL_TRACES = 6;
+/** Runs with fewer spans carry too little for the model to read. */
+const MIN_MODEL_SPANS = 3;
 async function diagnoseSpans(spans, context, options) {
 	if (context.subject !== "internal" && context.subject !== "customer") throw new TypeError(`diagnoseSpans: subject must be internal or customer, got ${String(context.subject)}`);
 	if (context.subject === "customer" && context.topology !== void 0) throw new TypeError("diagnoseSpans: topology analysis is internal only; a customer context must not carry one");
@@ -1124,17 +1163,17 @@ async function diagnoseSpans(spans, context, options) {
 		reason: options.reason ?? "deterministic mode: no model call was made"
 	});
 	else {
-		const maxTraces = options.model.maxTraces ?? 6;
+		const maxTraces = options.model.maxTraces ?? DEFAULT_MAX_MODEL_TRACES;
 		const ranked = rankTraces(traces);
 		const selected = ranked.slice(0, maxTraces);
 		const short = traces.size - ranked.length;
 		if (ranked.length === 0) skipped.push({
 			analysis: "model reading of the runs",
-			reason: `no run has the 3 spans the model needs to read it; ${traces.size} runs were too short`
+			reason: `no run has the ${MIN_MODEL_SPANS} spans the model needs to read it; ${traces.size} runs were too short`
 		});
 		else if (short > 0) skipped.push({
 			analysis: `model reading of ${short} of ${traces.size} runs`,
-			reason: `those runs have fewer than 3 spans, too few for the model to read`
+			reason: `those runs have fewer than ${MIN_MODEL_SPANS} spans, too few for the model to read`
 		});
 		if (ranked.length > selected.length) skipped.push({
 			analysis: `model reading of ${ranked.length - selected.length} of ${ranked.length} runs`,
@@ -1222,7 +1261,7 @@ async function diagnoseSpans(spans, context, options) {
 }
 /** Runs with more errors first, then larger runs; runs under MIN_MODEL_SPANS are left out. */
 function rankTraces(traces) {
-	return [...traces.entries()].filter(([, spans]) => spans.length >= 3).map(([traceId, spans]) => ({
+	return [...traces.entries()].filter(([, spans]) => spans.length >= MIN_MODEL_SPANS).map(([traceId, spans]) => ({
 		traceId,
 		spans,
 		errors: spans.filter((span) => span.status === "ERROR").length
@@ -1266,6 +1305,6 @@ function totalUsage(runs, pricing) {
 	};
 }
 //#endregion
-export { DIAGNOSIS_FINDINGS_SCHEMA_ID, diagnoseSpans, epochMillis, ingestSpans, isContentAttribute, nodeHttpPrimeBridgeTransport, rankFirstFailure, validateDiagnosisFindings };
+export { diagnoseSpans, epochMillis, ingestSpans, isContentAttribute, nodeHttpPrimeBridgeTransport, rankFirstFailure };
 
 //# sourceMappingURL=index.js.map