@tangle-network/agent-eval 0.216.1 → 0.217.1

package/dist/diagnosis/index.d.ts

@@ -1,8 +1,8 @@
-import { o as FailureBlame, s as FailureClassification } from "../failure-cluster-CbgEDcFJ.js";
-import { r as ExecutionReport } from "../analyze-runs-D68fd-su.js";
-import { l as PrimeRawUsage } from "../prime-protocol-HiB9HX7E.js";
+import { a as FailureClassification, i as FailureBlame } from "../failure-cluster-DN0t63Z0.js";
+import { r as ExecutionReport } from "../analyze-runs-DyeNcenW.js";
+import { l as PrimeRawUsage } from "../prime-protocol-BWG2DQIk.js";
 import { i as nodeHttpPrimeBridgeTransport, n as PrimeBridgeTransportRequest, r as PrimeBridgeTransportResult, t as PrimeBridgeTransport } from "../prime-bridge-transport-6feEglLf.js";
-import { a as isContentAttribute, i as ingestSpans, n as IngestReport, r as epochMillis, t as DiagnosisSpan } from "../spans-BHAUXMGN.js";
+import { a as isContentAttribute, i as ingestSpans, n as IngestReport, r as epochMillis, t as DiagnosisSpan } from "../spans-CVngGij3.js";
 import { SpanKind, TraceValidation } from "@tangle-network/agent-trace-contract";
 //#region src/diagnosis/findings.d.ts
 /**
@@ -13,7 +13,6 @@ import { SpanKind, TraceValidation } from "@tangle-network/agent-trace-contract"
  * objects, so a document that passes {@link validateDiagnosisFindings} passes
  * that schema.
  */
-export declare const DIAGNOSIS_FINDINGS_SCHEMA_ID = "https://tangle.tools/schemas/diagnosis-findings-v1.json";
 type DiagnosisSeverity = 'critical' | 'high' | 'medium' | 'low';
 /** `observed` is read directly off the trace; `inferred` is a model's reading of it. */
 type DiagnosisConfidence = 'observed' | 'inferred';
@@ -64,15 +63,6 @@ interface DiagnosisFindingsDocument {
   };
   findings: DiagnosisFinding[];
 }
-/**
- * Structural check against the v1 schema plus the two rules the schema states
- * in prose: an unavailable capability carries a reason, and a measure carries a
- * denominator. When `spanIds` is given, every evidence id must resolve to it.
- * Returns the list of defects; empty means valid.
- */
-export declare function validateDiagnosisFindings(document: unknown, options?: {
-  spanIds?: ReadonlySet<string>;
-}): string[];
 //#endregion
 //#region src/diagnosis/first-failure.d.ts
 type FirstFailureStage = 'error-span' | 'failed-outcome';

package/dist/diagnosis/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/diagnosis/findings.ts","../../src/diagnosis/first-failure.ts","../../src/diagnosis/deterministic.ts","../../src/diagnosis/model-pass.ts","../../src/diagnosis/engine.ts"],"mappings":";;;;;;;;;;;;;;;qBASa;KAGD;;KAGA;UAEK;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;EACV;EACA;EACA,UAAU;;EAEV;EACA;EACA,YAAY;EACZ;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,QAAQ;EACR;;UAGe;EACf;EACA;IACE;IACA;IACA;IACA;;EAEF;IACE,cAAc,eAAe;IAC7B,SAAS;IACT,YAAY;;EAEd,UAAU;;;;;;;;wBAiBI,0BACd,mBACA;EAAW,UAAU;;;;KCpCX;UAEK;EACf;EACA;EACA,OAAO;EACP;EACA;EACA,MAAM;;EAEN;;EAEA,gBAAgB;EAChB,OAAO;;EAEP;EACA;;UAGe;EACf;EACA;EACA,OAAO;;EAEP;EACA;EACA;;UAGe;EACf;EACA;EACA;;;;;;;UAQe;EACf;EACA;;EAEA;EACA;;KAGU,eACR,oBACA,wBACA,mBACA;;wBAGY,iBAAiB,gBAAgB,kBAAkB;;;UCjClD;EACf;EACA;EACA,aAAa;EACb;EACA;EACA;;EAEA,eAAe;;EAEf,WAAW;EACX;;UAGe;EACf,YAAY;EACZ,cAAc,eAAe;EAC7B,SAAS;EACT,OAAO;EACP,UAAU;EACV;;;;UCpEe;;EAEf,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IAAY;IAA4B;;;UAGzB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD,OAAO;EACP;;;;UC/Be;;EAEf;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;IAAa,MAAM;MAAQ;QAAe;MAA6B;;KAG7D;EACN;EAAuB;EAAiB,SAAS;;EACjD;EAAe,OAAO;EAAuB,SAAS;;UAE3C;EACf;EACA;EACA;EACA;;UAGe;;EAEf,UAAU;;EAEV,WAAW;;EAEX,OAAO;;EAEP,WAAW;IAAQ;IAAiB;;EACpC,OAAO;EACP,aAAa;EACb,QAAQ;;EAER,UAAU;EACV;IACE;IACA;IACA,MAAM;IACN;IACA,OAAO;;EAET;IAAU;IAA8B;;;wBAOpB,cACpB,2BACA,SAAS,kBACT,SAAS,mBACR,QAAQ"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/diagnosis/findings.ts","../../src/diagnosis/first-failure.ts","../../src/diagnosis/deterministic.ts","../../src/diagnosis/model-pass.ts","../../src/diagnosis/engine.ts"],"mappings":";;;;;;;;;;;;;;;KAWY;;KAGA;UAEK;EACf;EACA;EACA;;EAEA;;UAGe;EACf;EACA,UAAU;EACV;EACA;EACA,UAAU;;EAEV;EACA;EACA,YAAY;EACZ;;UAGe;EACf;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA,QAAQ;EACR;;UAGe;EACf;EACA;IACE;IACA;IACA;IACA;;EAEF;IACE,cAAc,eAAe;IAC7B,SAAS;IACT,YAAY;;EAEd,UAAU;;;;KChBA;UAEK;EACf;EACA;EACA,OAAO;EACP;EACA;EACA,MAAM;;EAEN;;EAEA,gBAAgB;EAChB,OAAO;;EAEP;EACA;;UAGe;EACf;EACA;EACA,OAAO;;EAEP;EACA;EACA;;UAGe;EACf;EACA;EACA;;;;;;;UAQe;EACf;EACA;;EAEA;EACA;;KAGU,eACR,oBACA,wBACA,mBACA;;wBAGY,iBAAiB,gBAAgB,kBAAkB;;;UCjClD;EACf;EACA;EACA,aAAa;EACb;EACA;EACA;;EAEA,eAAe;;EAEf,WAAW;EACX;;UAGe;EACf,YAAY;EACZ,cAAc,eAAe;EAC7B,SAAS;EACT,OAAO;EACP,UAAU;EACV;;;;UCpEe;;EAEf,WAAW;;EAEX;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IAAY;IAA4B;;;UAGzB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;;UAGe;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;IAAa;IAAiB;IAAkB;;EAChD,OAAO;EACP;;;;UC/Be;;EAEf;;EAEA;;EAEA;;;;;EAKA;;EAEA;;EAEA;IAAa,MAAM;MAAQ;QAAe;MAA6B;;KAG7D;EACN;EAAuB;EAAiB,SAAS;;EACjD;EAAe,OAAO;EAAuB,SAAS;;UAE3C;EACf;EACA;EACA;EACA;;UAGe;;EAEf,UAAU;;EAEV,WAAW;;EAEX,OAAO;;EAEP,WAAW;IAAQ;IAAiB;;EACpC,OAAO;EACP,aAAa;EACb,QAAQ;;EAER,UAAU;EACV;IACE;IACA;IACA,MAAM;IACN;IACA,OAAO;;EAET;IAAU;IAA8B;;;wBAOpB,cACpB,2BACA,SAAS,kBACT,SAAS,mBACR,QAAQ"}

package/dist/diagnosis/index.js

@@ -1,12 +1,15 @@
-import { r as summarizeExecution } from "../analyze-runs-C84irm4i.js";
-import { a as emptyRedactionReport, c as redactText, o as redact } from "../redact-B5xXKqQ3.js";
+import { r as summarizeExecution } from "../analyze-runs-DqPJaJ4X.js";
+import { a as redact, i as emptyRedactionReport, s as redactText } from "../redact-mtkTiJVk.js";
 import { t as packageVersion } from "../package-version-D7lQHt_-.js";
-import { o as classifyFailureReason } from "../failure-taxonomy-CjabTATj.js";
-import { c as primeProtocolSha256, d as runPrimeExchange, n as buildPrimePrompt } from "../prime-protocol-CHKgA3a6.js";
+import { r as classifyFailureReason } from "../failure-taxonomy-Bx5oWqaW.js";
+import { c as runPrimeExchange, n as buildPrimePrompt, o as primeProtocolSha256 } from "../prime-protocol-BFgGrEMP.js";
 import { t as nodeHttpPrimeBridgeTransport } from "../prime-bridge-transport-Z3cpKGoV.js";
-import { t as fromOtelSpans } from "../otel-spans-B9tUXRHF.js";
+import { t as fromOtelSpans } from "../otel-spans-CdN909k1.js";
 import { createHash, createHmac, randomBytes } from "node:crypto";
 import { MODEL_ATTR_KEYS, SPAN_KINDS, TOOL_NAME_ATTR_KEYS, resolveSpanKind, validateTraceSpans } from "@tangle-network/agent-trace-contract";
+//#region src/diagnosis/first-failure.ts
+/** Span ids listed beside a result; the count carries the rest. */
+const MAX_FIRST_FAILURE_IDS = 8;
 const OUTCOME_ATTR = "agent.outcome";
 /** Rank the first failure of one trace. Throws when the spans belong to more than one trace. */
 function rankFirstFailure(spans) {
@@ -73,7 +76,7 @@ function rank(traceId, stage, candidates) {
 		message: first.statusMessage,
 		classification,
 		blame: classification?.blame ?? "unknown",
-		later: later.slice(0, 8).map((span) => span.spanId),
+		later: later.slice(0, MAX_FIRST_FAILURE_IDS).map((span) => span.spanId),
 		laterCount: later.length
 	};
 }
@@ -83,7 +86,7 @@ function ambiguous(traceId, stage, candidates, reason) {
 		status: "ambiguous",
 		traceId,
 		stage,
-		candidates: ids.slice(0, 8),
+		candidates: ids.slice(0, MAX_FIRST_FAILURE_IDS),
 		candidateCount: ids.length,
 		reason
 	};
@@ -311,6 +314,28 @@ function firstString(attributes, keys) {
 function stringOr(value, fallback) {
 	return typeof value === "string" && value.length > 0 ? value : fallback;
 }
+//#endregion
+//#region src/diagnosis/deterministic.ts
+/**
+* The deterministic pass: everything the engine can say without a model.
+*
+* It produces the capability table (the trace contract's own validator, so the
+* reasons are the trace's reasons), execution facts with token and cost
+* accounting (agent-eval's `fromOtelSpans` + `summarizeExecution`), and
+* `observed` findings computed directly from span data. Every observed finding
+* carries a measure with its denominator and cites the spans it counted.
+*/
+/** Evidence ids cited per finding; the measure carries the full count. */
+const MAX_EVIDENCE_PER_FINDING = 8;
+/**
+* Identical tool calls with no different tool call between them that count as
+* a repeated-call run. Model turns between the calls do not break the run: an
+* agent that calls, reads, and calls again with the same input is the waste
+* this detects.
+*/
+const REPEAT_RUN_LENGTH = 3;
+/** A tool is reported only once it has at least this many calls. */
+const MIN_TOOL_CALLS = 3;
 /** The analysis each capability gates, as the report names it. */
 const CAPABILITY_ANALYSES = {
 	"token-accounting": "token accounting",
@@ -397,7 +422,7 @@ function toolFailureFindings(tools) {
 	const findings = [];
 	for (const [tool, calls] of byTool) {
 		const failed = calls.filter((span) => span.status === "ERROR");
-		if (calls.length < 3 || failed.length < 2) continue;
+		if (calls.length < MIN_TOOL_CALLS || failed.length < 2) continue;
 		const rate = failed.length / calls.length;
 		const severity = rate >= .5 && failed.length >= 5 ? "high" : rate >= .2 ? "medium" : "low";
 		findings.push({
@@ -411,7 +436,7 @@ function toolFailureFindings(tools) {
 				unit: "%",
 				denominator: `${calls.length} calls to ${tool}`
 			},
-			evidence: failed.slice(0, 8).map((span) => span.spanId),
+			evidence: failed.slice(0, MAX_EVIDENCE_PER_FINDING).map((span) => span.spanId),
 			confidence: "observed",
 			recommendation: `Read the failed ${tool} calls' status messages and fix the most common cause, or give the agent the precondition it keeps missing.`
 		});
@@ -429,22 +454,22 @@ function repeatedCallFindings(traces) {
 			const last = current[current.length - 1];
 			if (last && last.inputDigest === span.inputDigest && (last.toolName ?? last.name) === (span.toolName ?? span.name)) current.push(span);
 			else {
-				if (current.length >= 3) runs.push(current);
+				if (current.length >= REPEAT_RUN_LENGTH) runs.push(current);
 				current = [span];
 			}
 		}
-		if (current.length >= 3) runs.push(current);
+		if (current.length >= REPEAT_RUN_LENGTH) runs.push(current);
 	}
 	if (runs.length === 0) return [];
 	const wasted = runs.reduce((sum, run) => sum + run.length - 1, 0);
 	const tracesAffected = new Set(runs.map((run) => run[0].traceId)).size;
 	runs.sort((a, b) => b.length - a.length);
-	const evidence = runs.flatMap((run) => run.slice(0, 2).map((span) => span.spanId)).slice(0, 8);
+	const evidence = runs.flatMap((run) => run.slice(0, 2).map((span) => span.spanId)).slice(0, MAX_EVIDENCE_PER_FINDING);
 	const rate = wasted / digestCalls;
 	return [{
 		id: stableId("repeated-calls", String(runs.length)),
 		severity: rate >= .1 ? "high" : rate >= .03 ? "medium" : "low",
-		claim: `The agent made the same tool call with the same input 3 or more times with no other tool call in between, ${runs.length} times across ${tracesAffected} runs; the longest streak was ${runs[0].length} calls.`,
+		claim: `The agent made the same tool call with the same input ${REPEAT_RUN_LENGTH} or more times with no other tool call in between, ${runs.length} times across ${tracesAffected} runs; the longest streak was ${runs[0].length} calls.`,
 		consequence: `${wasted} calls repeated the previous tool call with the same input, which is work the agent paid for twice without new information.`,
 		measure: {
 			name: "repeated identical tool calls",
@@ -477,7 +502,7 @@ function failedRunFindings(traces) {
 			unit: "%",
 			denominator: `${traces.size} runs`
 		},
-		evidence: failedRoots.slice(0, 8).map((span) => span.spanId),
+		evidence: failedRoots.slice(0, MAX_EVIDENCE_PER_FINDING).map((span) => span.spanId),
 		confidence: "observed",
 		recommendation: "Group the failed runs by their root status message and fix the largest group first."
 	}];
@@ -556,15 +581,6 @@ function round(value) {
 }
 //#endregion
 //#region src/diagnosis/findings.ts
-/**
-* The diagnosis findings document: what the engine returns and what the
-* diagnosis report renders. It mirrors the kit's JSON Schema
-* (`https://tangle.tools/schemas/diagnosis-findings-v1.json`, owned by the
-* traces repository's diagnosis kit) field for field, including its closed
-* objects, so a document that passes {@link validateDiagnosisFindings} passes
-* that schema.
-*/
-const DIAGNOSIS_FINDINGS_SCHEMA_ID = "https://tangle.tools/schemas/diagnosis-findings-v1.json";
 const SEVERITIES$1 = /* @__PURE__ */ new Set([
 	"critical",
 	"high",
@@ -1102,6 +1118,29 @@ function shortString(value) {
 	if (!trimmed) return void 0;
 	return trimmed.length <= MAX_STRING ? trimmed : `${trimmed.slice(0, MAX_STRING)}…`;
 }
+//#endregion
+//#region src/diagnosis/engine.ts
+/**
+* The agent-failure diagnosis engine: flat spans plus context in, a findings
+* document out.
+*
+* It is a plain function. Its only side effect is the model call made through
+* the caller's transport, and in deterministic mode it makes none. It keeps no
+* storage, files no issues, and retains nothing: where the spans came from and
+* where the findings go belong to the caller. That is what lets one engine
+* serve both the internal daily mining run and a customer diagnosis while their
+* data paths stay separate.
+*
+* Order of operations, each step reading only the previous step's output:
+*   1. secret filter and content drop over every input (spans, focus, topology);
+*   2. the deterministic pass: capability table, execution facts, observed findings;
+*   3. in model mode, the prime-protocol pass over the highest-signal runs;
+*   4. evidence resolution: every cited id must be a span id of the input that
+*      no other run shares; findings left without evidence are rejected.
+*/
+const DEFAULT_MAX_MODEL_TRACES = 6;
+/** Runs with fewer spans carry too little for the model to read. */
+const MIN_MODEL_SPANS = 3;
 async function diagnoseSpans(spans, context, options) {
 	if (context.subject !== "internal" && context.subject !== "customer") throw new TypeError(`diagnoseSpans: subject must be internal or customer, got ${String(context.subject)}`);
 	if (context.subject === "customer" && context.topology !== void 0) throw new TypeError("diagnoseSpans: topology analysis is internal only; a customer context must not carry one");
@@ -1124,17 +1163,17 @@ async function diagnoseSpans(spans, context, options) {
 		reason: options.reason ?? "deterministic mode: no model call was made"
 	});
 	else {
-		const maxTraces = options.model.maxTraces ?? 6;
+		const maxTraces = options.model.maxTraces ?? DEFAULT_MAX_MODEL_TRACES;
 		const ranked = rankTraces(traces);
 		const selected = ranked.slice(0, maxTraces);
 		const short = traces.size - ranked.length;
 		if (ranked.length === 0) skipped.push({
 			analysis: "model reading of the runs",
-			reason: `no run has the 3 spans the model needs to read it; ${traces.size} runs were too short`
+			reason: `no run has the ${MIN_MODEL_SPANS} spans the model needs to read it; ${traces.size} runs were too short`
 		});
 		else if (short > 0) skipped.push({
 			analysis: `model reading of ${short} of ${traces.size} runs`,
-			reason: `those runs have fewer than 3 spans, too few for the model to read`
+			reason: `those runs have fewer than ${MIN_MODEL_SPANS} spans, too few for the model to read`
 		});
 		if (ranked.length > selected.length) skipped.push({
 			analysis: `model reading of ${ranked.length - selected.length} of ${ranked.length} runs`,
@@ -1222,7 +1261,7 @@ async function diagnoseSpans(spans, context, options) {
 }
 /** Runs with more errors first, then larger runs; runs under MIN_MODEL_SPANS are left out. */
 function rankTraces(traces) {
-	return [...traces.entries()].filter(([, spans]) => spans.length >= 3).map(([traceId, spans]) => ({
+	return [...traces.entries()].filter(([, spans]) => spans.length >= MIN_MODEL_SPANS).map(([traceId, spans]) => ({
 		traceId,
 		spans,
 		errors: spans.filter((span) => span.status === "ERROR").length
@@ -1266,6 +1305,6 @@ function totalUsage(runs, pricing) {
 	};
 }
 //#endregion
-export { DIAGNOSIS_FINDINGS_SCHEMA_ID, diagnoseSpans, epochMillis, ingestSpans, isContentAttribute, nodeHttpPrimeBridgeTransport, rankFirstFailure, validateDiagnosisFindings };
+export { diagnoseSpans, epochMillis, ingestSpans, isContentAttribute, nodeHttpPrimeBridgeTransport, rankFirstFailure };
 
 //# sourceMappingURL=index.js.map