@tangle-network/agent-eval 0.140.1 → 0.141.0

package/dist/{default-registry-BOiBXNFh.js → default-registry-BupFSXSN.js}

@@ -1,7 +1,7 @@
 import { n as LlmClient } from "./llm-client-B3WXSH5Y.js";
 import { LLM_CONTEXT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_OUTPUT_TOKEN_ATTR_KEYS, TOOL_NAME_ATTR_KEYS } from "./trace-attributes.js";
 import { t as executionTrackByLane } from "./execution-tracks-CpgFPpS5.js";
-import { $ as spanEpochMillis, H as snapshotExactExecutionPlan, R as findingSubjectGrammarPromptFor, U as deepFreezeCanonicalJson, V as snapshotExactExecutionComponentIdentity, t as createTraceAnalyst } from "./kind-factory-DB7nIs35.js";
+import { $ as spanEpochMillis, H as snapshotExactExecutionPlan, R as findingSubjectGrammarPromptFor, U as deepFreezeCanonicalJson, V as snapshotExactExecutionComponentIdentity, t as createTraceAnalyst } from "./kind-factory-Bvwe3pup.js";
 import { i as validateUsageSettlementTimeout, o as makeFinding, t as assertValidAnalystUsageReceipt } from "./usage-receipt-CgxMEBZq.js";
 import { i as hashCanonical, r as canonicalString } from "./canonical-D011XM8r.js";
 import { t as analyzeSupervisorRunIntegrity } from "./integrity-hruKe6oe.js";
@@ -103,6 +103,27 @@ function resolveModel(req, defaultModel) {
 const INPUT_GROWTH_FACTOR = 3;
 /** Tool-usage signals need at least this many calls to be meaningful. */
 const MIN_TOOL_CALLS = 3;
+/**
+* Serial calls required before a strictly-decreasing output run counts as decay.
+*
+* A run of n independent lengths is strictly decreasing by chance with
+* probability 1/n!, so the old minimum of 3 fired on roughly one sequence in
+* six. The paired `inputIsMonotonic && inputGrew` guard does not offset that:
+* context accumulates by construction in a serial agent loop, so input growth
+* is very nearly free evidence. At 5 calls chance alone accounts for under 1%,
+* which is the bar a signal reported at full confidence has to clear.
+*/
+const OUTPUT_DECAY_MINIMUM_CALLS = 5;
+/**
+* The last output must fall to at most this fraction of the first.
+*
+* Length wanders between turns for reasons that are not degradation, so
+* direction alone is not a finding — an observed 784 → 646 run (18%) was
+* reported as decay and was noise. Requiring the response to lose most of its
+* length keeps the signal on the failure it names: late steps that quietly
+* stop doing the work.
+*/
+const OUTPUT_DECAY_MAXIMUM_RETAINED_FRACTION = .6;
 /** Tool names that read or check state count as self-verification, not mutation.
 *  Covers the inspect verbs plus the read/search tools real harnesses use to
 *  verify (Claude Code Read/Grep/Glob, codex read_file/ls/cat, git status/diff,
@@ -400,13 +421,14 @@ function tokenSignals(sequence) {
 			});
 		}
 	}
-	if (inputs.length >= 3 && inputs.length === outputs.length && inputs.every((value) => value !== null) && outputs.every((value) => value !== null)) {
+	if (inputs.length >= OUTPUT_DECAY_MINIMUM_CALLS && inputs.length === outputs.length && inputs.every((value) => value !== null) && outputs.every((value) => value !== null)) {
 		const first = outputs[0];
 		const last = outputs[outputs.length - 1];
 		const inputIsMonotonic = everyAdjacent(inputs, (previous, current) => current >= previous);
 		const outputIsMonotonic = everyAdjacent(outputs, (previous, current) => current <= previous);
 		const inputGrew = inputs[inputs.length - 1] > inputs[0];
-		if (inputIsMonotonic && inputGrew && outputIsMonotonic && last < first) signals.push({
+		const decayIsMaterial = last <= first * OUTPUT_DECAY_MAXIMUM_RETAINED_FRACTION;
+		if (inputIsMonotonic && inputGrew && outputIsMonotonic && decayIsMaterial) signals.push({
 			code: "output-length-decay",
 			severity: "medium",
 			detail: `LLM output tokens shrank ${first}→${last} over ${outputs.length} serial calls while input tokens increased monotonically.`,
@@ -624,30 +646,53 @@ var ControlIntegrityAnalyst = class {
 const CONTROL_INTEGRITY_ANALYST = new ControlIntegrityAnalyst();
 const FAILURE_MODE_KIND_SPEC = {
 	id: "failure-mode",
-	description: "Clusters trace-dataset failures into distinct failure modes with cited evidence and a short recommended action.",
+	description: "Ranks failures by the work each one burned — spans, model calls, and wall-clock between the error and the span where the agent resumed — and treats a failure that reached a human as the most expensive class.",
 	area: "failure-mode",
-	version: "1.2.0",
-	instructions: `You are a failure-mode classifier for an OTLP trace dataset. Your job is to identify the **distinct ways agents failed** in this dataset, not to grade individual runs.
+	version: "2.0.0",
+	instructions: `You are a failure-cost analyst for an OTLP trace dataset. Your job is to find the failures that **burned the most agent work** and to price each one in spans, model calls, and wall-clock. How often an error occurred is not its cost and is not your output.
 
 ${findingSubjectGrammarPromptFor("failure-mode")}
 
-DISCOVERY → CLUSTER → CITE protocol:
+Use a lowercase cluster label that names the cost shape, not the error string: "human-corrected-file-overwrite", "unrecovered-auth-loop", "long-detour-schema-guess".
 
-1. Call \`getDatasetOverview({})\` first. Use \`has_errors\`, \`models\`, \`agent_names\`, \`tools\`, and \`sample_trace_ids\` to size the failure surface.
-2. Use \`queryTraces(filters={"has_errors": true}, limit=...)\` to pull error-bearing traces. Combine with \`countTraces\` to see what fraction of the dataset failed.
-3. For each candidate failure cluster, use \`searchTrace\` with regex like \`STATUS_CODE_ERROR\`, \`MaxTurnsExceeded\`, \`assertion\`, \`unauthorized\`, \`timeout\`, \`429\`, \`5\\d\\d\`, the agent's specific error strings, or the names of its tools. Pull one or two representative traces per cluster, **not all** of them.
-4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
-5. For each defensible cluster, emit ONE finding. Use a lowercase cluster label matching the subject grammar ("tool-call-loop", "auth-revoked-mid-run", ...). Rate it critical when it blocks the run, high when the run finishes degraded, and medium when it slows convergence. Cite representative spans and include exact error, payload, or contradictory-output quotes. Use confidence 0.85+ when multiple traces show the same shape, 0.6-0.8 for a single-trace inference, and <0.5 for speculation. Keep the imperative fix idea short; the improvement analyst expands it.
+WORKLIST → RECOVERY BOUNDARY → PRICE → CITE protocol:
 
-If the dataset has no failures, return an empty findings array — do NOT pad with low-confidence speculation.
+1. \`getDatasetOverview({})\` first. Read \`errors.trace_count\`, \`errors.span_count\`, and \`error_clusters[]\` — each cluster carries \`signature\`, \`tool_name\`, \`trace_count\`, \`span_count\`, \`prevalence\`, \`exemplar_trace_ids\`, and \`exemplar_span_ids\`. This is the deterministic analyzer's output, already computed for free. It is your WORKLIST: the exemplar ids name the spans you go price. It is never your findings.
+2. Reach error-bearing traces the exemplars miss with \`queryTraces(filters={"has_errors": true}, limit=...)\`. Prefer long traces and traces whose errors sit early: a failure has room to be expensive only when work followed it.
+3. Take one candidate error span E and read its trace in order. \`viewTrace\` returns that trace's spans sorted by \`start_time\`, which is the ordering every distance below is counted in. When \`viewTrace\` returns \`oversized\` instead of spans, locate candidates with \`searchTrace\` and pull the exact ids with \`viewSpans\`; use \`searchSpan\` when one span's payload is truncated.
+4. Find the RECOVERY BOUNDARY R — the first span after E in which the agent resumed productive work. Productive means a different, task-advancing action. A retry of the same call with the same arguments, a re-read of the same file, a re-plan of the same step, and an apology are all still inside the failure. Then price it:
+   - spans_burned: spans strictly between E and R
+   - model_calls_burned: how many of those spans have kind \`LLM\`
+   - wall_clock_ms: R \`end_time\` minus E \`start_time\`
+   Report all three. If the agent never resumed, R is the trace's last span and the failure is UNRECOVERED.
+5. Set severity from what ENDED the failure and from the measured distance, never from how many times the signature appears:
+   - HUMAN-CORRECTED — a human turn after E supplies the correction. This is the most expensive class at any span count, because the failure escaped the agent entirely and reached the user: severity critical. Detect it as a new user-role message in a later \`LLM\` span's input attributes, or a new root-level \`AGENT\` span in which the user restates or repairs the task. Search with patterns like \`"role"\\s*:\\s*"user"\`, \`(that|this) is (wrong|not what)\`, \`I (said|asked|told you)\`, \`stop\`, \`undo\`, \`revert\`, \`you (deleted|broke|missed|ignored)\`, \`try again\`. Quote the human's words exactly.
+   - UNRECOVERED — the trace ends without resumption: severity critical.
+   - LONG DETOUR — 10 or more spans burned, or 3 or more \`LLM\` spans burned: severity high.
+   - SHORT DETOUR — 2 to 9 spans burned: severity medium.
+   - SELF-RECOVERED ON THE NEXT SPAN — cost is approximately zero. Do not emit it. That is resilience, and reporting it dilutes the ranking.
+   One error that cost 20 spans outranks 50 errors that each cost one retry. Frequency may appear in a finding only as a multiplier on a measured per-instance cost, never as the reason for its severity.
+6. **Cluster, do not enumerate.** Errors sharing a root cause AND a recovery shape are ONE finding: report the summed cost across instances, state how many instances it covers, and cite the bounding pair of the single most expensive instance. Two errors with the same signature but different recovery shapes — one shrugged off, one human-corrected — are NOT the same finding; the expensive one is the finding and the cheap one is noise.
 
-**Use subqueries over loaded evidence.** After the first scan, load representative span excerpts for each candidate cluster. Then send one bounded \`llm_query\` per cluster in one batch, including the exact excerpts and asking it to classify the root cause. Subqueries cannot call trace tools. Merge or split clusters yourself from their classifications and the cited source evidence.`,
+FORBIDDEN OUTPUT. Each of the following duplicates the deterministic pass, and a duplicate finding is worse than no finding because it costs a reviewer the same attention while carrying no new information:
+   - a count of errors by tool ("Bash is the dominant failure surface, 54 error spans")
+   - a count or inventory of error signatures ("47 distinct error signatures")
+   - an error rate ("79 of 5260 tool calls ended in errors")
+   - any restatement of \`prevalence\`, \`trace_count\`, or \`span_count\` from \`error_clusters[]\`
+Every finding MUST state its measured cost — spans burned, model calls burned, wall-clock — in the claim, and MUST cite the two spans that bound it: the error span and the recovery span, or the escaping human turn, or the trace's last span when unrecovered. A failure you cannot bound with two spans is one you did not measure: drop it.
+
+**Adjudicate boundaries with subqueries.** The single judgement call in this protocol is whether a post-error span is genuine resumption or more flailing. Load E, the spans between, and the candidate R, then send one bounded \`llm_query\` per candidate carrying those exact excerpts and asking which span first advances the task. Subqueries cannot call trace tools, so a trace id tells them nothing — paste the excerpts. Accept a boundary only when the excerpts you loaded support the answer.
+
+Confidence 0.9+ when both bounding spans are quoted and the spans between them were counted directly; 0.6-0.8 when an oversized trace forced you to sample the interval and the distance is an estimate; below 0.5 does not belong in this analyst, because an unmeasurable cost is not a finding. Keep the recommended action a short imperative; the improvement analyst expands it.
+
+If every error in this dataset was recovered on the next span, return an empty findings array. That is the correct answer for a dataset whose failures were all cheap. Do not backfill it with the counts listed above and do not pad it with speculation.`,
 	toolGroup: "all",
 	limits: {
 		maxLlmCalls: 8,
 		maxIterations: 24,
-		maxToolCalls: 64
-	}
+		maxToolCalls: 80
+	},
+	minimumEvidenceCitations: 2
 };
 const IMPROVEMENT_KIND_SPEC = {
 	id: "improvement",
@@ -691,6 +736,44 @@ Do NOT propose a fix you cannot defend with evidence. "Tighten the prompt" is no
 		maxOutputChars: 12e3
 	}
 };
+const INTENT_DIVERGENCE_KIND_SPEC = {
+	id: "intent-divergence",
+	description: "Anchors each corrective human turn to the earliest assistant turn where the divergence from stated intent was already detectable, and prices it in burned turns.",
+	area: "intent-divergence",
+	version: "1.0.0",
+	instructions: `You are an intent-divergence analyst for an OTLP trace dataset. Your job is to find where the agent stopped doing what the user asked, and to establish **how early that was knowable**.
+
+This dataset carries ground truth the other analysts do not have. When a human turn corrects the agent — "no, I meant…", "stop doing X", "why are you still asking" — the user has labelled a divergence for you. That label marks the END of the divergence, not its start. Your finding is the start: the earliest assistant turn already off-intent, the signal in the user's stated intent it contradicted, and the number of turns burned before the correction landed.
+
+${findingSubjectGrammarPromptFor("intent-divergence")}
+
+DISCOVERY → BACKTRACK → QUANTIFY → CITE protocol:
+
+1. \`getDatasetOverview({})\` first. Note the agent names and \`sample_trace_ids\`; conversational datasets carry several human turns per trace and those are the ones worth pulling.
+2. **DISCOVERY — find the corrective human turns.** Use \`searchTrace\` for user-authored corrections:
+   Every pattern MUST begin with \`(?i)\`. The trace store compiles a search pattern case-sensitively unless it opens with that flag, and corrections are overwhelmingly sentence-initial and capitalised ("Stop", "No,", "I said"), so a lowercase pattern silently returns zero hits.
+   - Imperative stops: \`(?i)\\bstop\\b\`, \`(?i)\\b(don'?t|do not)\\b\`, \`(?i)^\\s*no[,.!]\`, \`(?i)\\bnot what I (wanted|asked|meant)\\b\`, \`(?i)\\bthat'?s not what\\b\`, \`(?i)\\brevert\\b\`, \`(?i)\\bstart over\\b\`
+   - Repetition and frustration: \`(?i)\\bI (said|asked|told you)\\b\`, \`(?i)\\balready told you\\b\`, \`(?i)\\bwhy are you (still )?\\w+ing\\b\`, \`(?i)\\bjust (do|use|write|make)\\b\`, \`(?i)\\bagain\\b\`
+   - Clarification markers (second-order, see step 6): \`(?i)\\bI mean(t)?\\b\`, \`(?i)\\bto be clear\\b\`, \`(?i)\\bactually,? I (want|need)\\b\`, \`(?i)\\bwhat I meant\\b\`, \`(?i)\\blet me rephrase\\b\`
+   Attribute every match to a role before using it. The same phrases inside an assistant turn are hedging, not correction.
+3. **BACKTRACK — walk the trace backward from the correction.** Read the ordered turns preceding it with \`viewTrace\` / \`viewSpans\`. Locate the user's ORIGINAL statement of intent (usually the first human turn, sometimes a constraint added mid-run), then locate the FIRST assistant turn that contradicts it. That turn, not the correction, anchors the finding. Name the concrete signal already available at that turn: an explicit constraint the agent violated, a scope it exceeded, a different question it answered, a file or target the user never named, a decision the user had already made.
+4. **QUANTIFY — count the burned turns.** K = (index of the corrective human turn) − (index of the earliest detectable assistant turn). Report both indices and K. If you cannot locate an earliest-detectable turn, you have a correction and not a divergence: drop it.
+5. **CLUSTER.** Repeated corrections about the same misread intent are ONE finding citing all of them, not one finding per corrective turn. Corrections about genuinely different intents in the same trace are separate findings.
+6. **Second-order — the request itself was ambiguous.** When the user CLARIFIES rather than corrects ("I mean the staging config, not prod"), the agent's reading was defensible and the request was underspecified or self-contradictory. Emit those against a \`scaffolding:*\` locus — a clarifying-question policy or a plan checkpoint — because the fix is one sharp question before acting, priced against the K turns the ambiguity cost. Do not charge the agent's instructions for a request that could not be resolved from its own text.
+
+**Test competing anchors with subqueries.** After the backward walk, load the excerpts for the original intent turn, your candidate earliest-detectable turn, and the turn before it. Send one bounded \`llm_query\` per candidate anchor asking whether the divergence is already present in that turn's text alone. Subqueries cannot call trace tools, and a turn index alone is insufficient context. Take the EARLIEST candidate the loaded excerpts support.
+
+For each cluster, emit ONE finding. Use an exact locus from the subject grammar — the surface whose edit prevents the divergence, not the surface where it surfaced. State the claim in the shape "turn N did X against stated intent Y; user corrected at turn N+K, burning K turns." Rate it critical when the divergence produced a user-visible or irreversible action, high when K is 3 or more or the run ended without the intent satisfied, medium for one or two burned turns, and low for a cosmetic correction. Cite BOTH spans with exact quotes: the earliest-detectable assistant turn and the corrective human turn. Use confidence 0.85+ when the stated intent and the diverging turn are both quotable and the contradiction is explicit, and 0.6-0.8 when the earliest-detectable turn is inferred from surrounding context. The recommended action must be the literal instruction, question, or checkpoint to add.
+
+Do NOT report a divergence the agent noticed and reversed inside the same turn — that is self-correction and it cost the user nothing. Do NOT report a human turn that adds NEW scope; a changed mind is not a divergence. If the dataset holds no corrective or clarifying human turns, return an empty findings array instead of grading tone.`,
+	toolGroup: "all",
+	limits: {
+		maxLlmCalls: 6,
+		maxIterations: 22,
+		maxToolCalls: 64
+	},
+	minimumEvidenceCitations: 2
+};
 const KNOWLEDGE_GAP_KIND_SPEC = {
 	id: "knowledge-gap",
 	description: "Identifies missing or stale pieces of knowledge — primarily against the agent-knowledge wiki — and attributes each to the runtime layer (wiki page, claim, raw source, websearch, tool-doc, system-prompt, memory) that should have held it.",
@@ -768,12 +851,17 @@ Do NOT report a finding if the agent caught and corrected the false belief in th
 //#endregion
 //#region src/analyst/kinds/index.ts
 /**
-* The default kind suite. Order is the run order operators should
-* use: failure-mode first (no upstream deps), gap + poisoning next
-* (both depend on failures), improvement last (chains all three).
+* The default kind suite. Order is the run order operators should use:
+* failure-mode and intent-divergence first (neither reads upstream
+* findings), gap + poisoning next (both explain the problems those two
+* found), improvement last (chains all four). Intent-divergence sits
+* ahead of improvement deliberately — a proposed edit should be able to
+* act on a priced divergence, and it is the only kind whose findings
+* carry a burned-turn cost to rank against.
 */
 const DEFAULT_TRACE_ANALYST_KINDS = [
 	FAILURE_MODE_KIND_SPEC,
+	INTENT_DIVERGENCE_KIND_SPEC,
 	KNOWLEDGE_GAP_KIND_SPEC,
 	KNOWLEDGE_POISONING_KIND_SPEC,
 	IMPROVEMENT_KIND_SPEC
@@ -2361,4 +2449,4 @@ function buildDefaultAnalystRegistry(options = {}) {
 //#endregion
 export { createChatClient as C, computeTraceMetrics as S, CONTROL_INTEGRITY_ANALYST as _, analystFindingDigest as a, behavioralAnalyst as b, completedAnalystReviewQuality as c, validateAnalystReviewDecisions as d, DEFAULT_TRACE_ANALYST_KINDS as f, FAILURE_MODE_KIND_SPEC as g, IMPROVEMENT_KIND_SPEC as h, assertExactRegistryRunOpts as i, readAnalystReview as l, KNOWLEDGE_GAP_KIND_SPEC as m, AnalystRegistry as n, analystRunDigest as o, KNOWLEDGE_POISONING_KIND_SPEC as p, ExactAnalystRunExecutionError as r, assertUniqueFindingIds as s, buildDefaultAnalystRegistry as t, snapshotAnalystRun as u, ControlIntegrityAnalyst as v, deriveEfficiencyFindings as x, emitControlIntegrityFindings as y };
 
-//# sourceMappingURL=default-registry-BOiBXNFh.js.map
+//# sourceMappingURL=default-registry-BupFSXSN.js.map