@opensearch-project/agent-health 0.5.2 → 0.7.0

package/lib/dist/lib/matchers/judgeReasoningParse.js

@@ -0,0 +1,128 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/** Map a matched verdict keyword onto the canonical kind. */
+function verdictKind(keyword) {
+    const k = keyword.toUpperCase();
+    if (k.includes('CONTRADICT') && !k.includes('MISSING'))
+        return 'contradicted';
+    if (k.includes('MISSING') || k.includes('NOT STATED'))
+        return 'missing';
+    if (k.includes('PARTIAL'))
+        return 'partial';
+    return 'stated';
+}
+// Verdict keywords the judge writes inline after a fact. Order matters:
+// longest / most specific first so "MISSING/CONTRADICTED" isn't split.
+const VERDICT_KEYWORD = /(MISSING\s*\/\s*CONTRADICTED|FULLY\s+STATED|PARTIALLY\s+STATED|NOT\s+STATED|CONTRADICTED|MISSING|PARTIAL(?:LY)?)/i;
+// Parse at most this much reasoning — real verdicts are a few KB; a hard cap
+// bounds regex work on pathological inputs (defense against backtracking
+// blowups on adversarial multi-hundred-KB strings).
+const MAX_PARSE_CHARS = 20_000;
+// A fact item: list marker (`1.`, `1)`, `**Fact 1:`, `Required fact 1 (`),
+// then the fact text (quoted or plain), a separator (—, -, :, en-dash, `):`),
+// then the verdict keyword. Fact text is capped to keep matches sane.
+// NOTE: deliberately no lookbehind — constructed-at-import regexes with
+// lookbehind hard-crash report rendering on older WebKit. The leading
+// whitespace/newline is consumed instead (harmless: markers never overlap).
+const FACT_ITEM = new RegExp(String.raw `(?:^|[\s\n])` + // start of string or after whitespace (inline numbered lists)
+    String.raw `(?:\*\*)?(?:Required\s+fact\s+\d+|Fact\s+\d+|\d+)\s*[.):\u2013\u2014-]?\s*` + // marker
+    String.raw `(?:\*\*)?\s*` +
+    String.raw `['‘"“(]?(.{4,240}?)['’"”)]?` + // fact text (lazy)
+    String.raw `(?:\*\*)?\s*[\u2013\u2014:(\u2015-]+\s*(?:\*\*)?\s*` + // separator
+    VERDICT_KEYWORD.source + // verdict
+    String.raw `(?:\s+stated)?` + // "PARTIALLY stated"
+    String.raw `[.!]?\s*` +
+    // Trailing note: lazy, stopped by the NEXT numbered/bold fact item on the
+    // same line (inline lists put every fact in one paragraph) or line end.
+    String.raw `([^\n]*?)(?=\s\d+[.)]\s*['‘"“(*]|\s\*\*(?:Required\s+fact|Fact)|\n|$)`, 'gi');
+/**
+ * Extract per-required-fact verdicts from judge reasoning prose.
+ * Returns `[]` when nothing that looks like a fact list is present —
+ * callers must fall back to showing the raw reasoning.
+ */
+export function parseFactVerdicts(reasoning) {
+    if (!reasoning || reasoning.length < 20)
+        return [];
+    const text = reasoning.slice(0, MAX_PARSE_CHARS);
+    const out = [];
+    const seen = new Set();
+    for (const m of text.matchAll(FACT_ITEM)) {
+        const factRaw = (m[1] ?? '').trim();
+        const keyword = m[2] ?? '';
+        // Guard against summary-phrase false positives like
+        // "4 facts fully stated (1.0 each)": require real fact text, not a
+        // recap that itself talks about facts/statements in aggregate.
+        if (!factRaw || factRaw.length < 8)
+            continue;
+        if (/^facts?\b/i.test(factRaw) || /\bfacts fully stated\b/i.test(factRaw))
+            continue;
+        // Strip markdown/bold leftovers and trailing separators.
+        const fact = factRaw.replace(/\*\*/g, '').replace(/[\u2013\u2014:-]+$/, '').trim();
+        const key = fact.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        let note = (m[3] ?? '').replace(/\*\*/g, '').trim().slice(0, 220);
+        // Notes that are just the start of the next list item are noise.
+        if (/^\d+[.)]/.test(note))
+            note = '';
+        out.push({ fact, verdict: verdictKind(keyword), ...(note ? { note } : {}) });
+        if (out.length >= 12)
+            break; // sanity cap
+    }
+    return out;
+}
+// Hex-ish document/article ids (8+ chars, at least one a–f so bare integers
+// like "10000000" never qualify) — what RAG corpora and judge reasoning use
+// when naming expected vs cited sources.
+const HEX_ID_SCAN = /\b(?=[0-9]*[a-f])[0-9a-f]{8,64}\b/i;
+/** First hex id within `window` chars after `index` in `text`, if any. */
+function firstIdAfter(text, index, window = 140) {
+    const m = text.slice(index, index + window).match(HEX_ID_SCAN);
+    return m?.[0];
+}
+/**
+ * Detect an "expected source X but cited/retrieved Y" statement.
+ * Conservative: both ids must be present and differ.
+ */
+export function parseSourceMismatch(reasoning) {
+    if (!reasoning)
+        return null;
+    const text = reasoning.slice(0, MAX_PARSE_CHARS);
+    // Expected id: first hex id shortly after an "expected source …" mention.
+    // (A character-class "gap" can't work here — prose like "document is
+    // article" contains hex letters — so scan a window for the first id.)
+    const expectedKw = text.match(/expected\s+source\s+(?:document|article)?/i);
+    if (!expectedKw || expectedKw.index === undefined)
+        return null;
+    const expected = firstIdAfter(text, expectedKw.index + expectedKw[0].length);
+    if (!expected)
+        return null;
+    // Cited id: first differing hex id shortly after a cite/retrieve verb.
+    const citedKw = /(?:\bcited\b|\bretrieved\b|\busing\s+article\b|\bcites\b)/gi;
+    for (const m of text.matchAll(citedKw)) {
+        if (m.index === undefined)
+            continue;
+        const id = firstIdAfter(text, m.index + m[0].length);
+        if (id && id.toLowerCase() !== expected.toLowerCase()) {
+            return { expected, cited: id };
+        }
+    }
+    // Compact form: "(b6c9353c vs 49d9e88f)" — either order relative to expected.
+    const vs = text.match(new RegExp(String.raw `([0-9a-f]{8,64})\s*(?:vs\.?|versus)\s*([0-9a-f]{8,64})`, 'i'));
+    if (vs) {
+        const [a, b] = [vs[1], vs[2]];
+        const other = a.toLowerCase() === expected.toLowerCase() ? b : a;
+        if (other && other.toLowerCase() !== expected.toLowerCase()) {
+            return { expected, cited: other };
+        }
+    }
+    return null;
+}
+/** Shorten a long doc id for display: `49d9e88fadbf…` (first 8 chars). */
+export function shortId(id) {
+    return id.length > 12 ? `${id.slice(0, 8)}…` : id;
+}
+//# sourceMappingURL=judgeReasoningParse.js.map

package/lib/dist/lib/matchers/judgeReasoningParse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judgeReasoningParse.js","sourceRoot":"","sources":["../../../matchers/judgeReasoningParse.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,6DAA6D;AAC7D,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAChC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,cAAc,CAAC;IAC9E,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,wEAAwE;AACxE,uEAAuE;AACvE,MAAM,eAAe,GACnB,mHAAmH,CAAC;AAEtH,6EAA6E;AAC7E,yEAAyE;AACzE,oDAAoD;AACpD,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,2EAA2E;AAC3E,8EAA8E;AAC9E,sEAAsE;AACtE,wEAAwE;AACxE,sEAAsE;AACtE,4EAA4E;AAC5E,MAAM,SAAS,GAAG,IAAI,MAAM,CAC1B,MAAM,CAAC,GAAG,CAAA,cAAc,GAAG,8DAA8D;IACvF,MAAM,CAAC,GAAG,CAAA,4EAA4E,GAAG,SAAS;IAClG,MAAM,CAAC,GAAG,CAAA,cAAc;IACxB,MAAM,CAAC,GAAG,CAAA,6BAA6B,GAAG,mBAAmB;IAC7D,MAAM,CAAC,GAAG,CAAA,qDAAqD,GAAG,YAAY;IAC9E,eAAe,CAAC,MAAM,GAAG,UAAU;IACnC,MAAM,CAAC,GAAG,CAAA,gBAAgB,GAAG,qBAAqB;IAClD,MAAM,CAAC,GAAG,CAAA,UAAU;IACpB,0EAA0E;IAC1E,wEAAwE;IACxE,MAAM,CAAC,GAAG,CAAA,uEAAuE,EACnF,IAAI,CACL,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAA6B;IAC7D,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACjD,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,oDAAoD;QACpD,mEAAmE;QACnE,+DAA+D;QAC/D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC7C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,SAAS;QACpF,yDAAyD;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClE,iEAAiE;QACjE,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,IAAI,GAAG,EAAE,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7E,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE;YAAE,MAAM,CAAC,aAAa;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4EAA4E;AAC5E,4EAA4E;AAC5E,yCAAyC;AACzC,MAAM,WAAW,GAAG,oCAAoC,CAAC;AAEzD,0EAA0E;AAC1E,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa,EAAE,MAAM,GAAG,GAAG;IAC7D,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/D,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA6B;IAC/D,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IAEjD,0EAA0E;IAC1E,qEAAqE;IACrE,sEAAsE;IACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAC5E,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/D,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,uEAAuE;IACvE,MAAM,OAAO,GAAG,6DAA6D,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,SAAS;QACpC,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YACtD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CACnB,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,wDAAwD,EAAE,GAAG,CAAC,CACpF,CAAC;IACF,IAAI,EAAE,EAAE,CAAC;QACP,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5D,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,OAAO,CAAC,EAAU;IAChC,OAAO,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AACpD,CAAC"}

package/lib/dist/lib/matchers/traces.d.ts

@@ -18,11 +18,26 @@
  * accessor.
  */
 import type { Span } from '../../types/index.js';
+/**
+ * Where `totalCost` came from:
+ *   - `'reported'` — every dollar came from a real cost attribute on a span.
+ *   - `'computed'` — no span carried a cost attribute; the whole total was
+ *     derived from token counts via the fallback pricing table.
+ *   - `'mixed'`    — some spans reported cost, others were computed.
+ *   - `'none'`     — no LLM spans, or no cost could be determined at all.
+ */
+export type CostSource = 'reported' | 'computed' | 'mixed' | 'none';
 export interface TracesAccessor {
-    /** Sum of all `gen_ai.usage.prompt_tokens + completion_tokens` across LLM spans. */
+    /** Sum of all prompt/completion/cache token-usage attrs across LLM spans. */
     totalTokens: number;
-    /** Sum of all `gen_ai.usage.cost_usd` (or derived) across LLM spans. */
+    /**
+     * Sum of all reported cost attrs, falling back to a computed estimate
+     * (from token counts + the model pricing table) when no span reports
+     * cost directly. See `costSource` to tell which happened.
+     */
     totalCost: number;
+    /** Whether `totalCost` was reported by spans, computed, both, or neither. */
+    costSource: CostSource;
     /** Tool invocation summary derived from spans. */
     toolCalls: ReadonlyArray<{
         name: string;

package/lib/dist/lib/matchers/traces.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../../matchers/traces.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAEjD,MAAM,WAAW,cAAc;IAC7B,oFAAoF;IACpF,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,SAAS,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/D,+EAA+E;IAC/E,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,6BAA6B;IAC7B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,cAAc,CAQpD;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAWxE;AAED,wDAAwD;AACxD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,cAAc,CA4CjE"}
+{"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../../matchers/traces.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAGjD;;;;;;;GAOG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;AAEpE,MAAM,WAAW,cAAc;IAC7B,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,UAAU,EAAE,UAAU,CAAC;IACvB,kDAAkD;IAClD,SAAS,EAAE,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC/D,+EAA+E;IAC/E,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IACnC,6BAA6B;IAC7B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,cAAc,CASpD;AAED;;;;;;;;;GASG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,cAAc,CAYxE;AAgCD,wDAAwD;AACxD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,cAAc,CA+IjE"}

package/lib/dist/lib/matchers/traces.js

@@ -2,6 +2,7 @@
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
  */
+import { lookupModelRates, computeCost } from './tracesPricing.js';
 /**
  * Empty accessor — silent zeros. Used when the agent opted out of traces
  * (`useTraces: false`). Reading any accessor returns `0` / `[]` so user
@@ -11,6 +12,7 @@ export function emptyTracesAccessor() {
     return {
         totalTokens: 0,
         totalCost: 0,
+        costSource: 'none',
         toolCalls: [],
         spanDuration: () => 0,
         spans: [],
@@ -33,48 +35,166 @@ export function unavailableTracesAccessor(reason) {
     return {
         get totalTokens() { return fail(); },
         get totalCost() { return fail(); },
+        get costSource() { return fail(); },
         get toolCalls() { return fail(); },
         get spans() { return fail(); },
         spanDuration(_name) { return fail(); },
     };
 }
+const PROMPT_TOKEN_KEYS = [
+    'gen_ai.usage.prompt_tokens',
+    'gen_ai.usage.input_tokens',
+    'llm.usage.prompt_tokens',
+    'input_tokens',
+];
+const COMPLETION_TOKEN_KEYS = [
+    'gen_ai.usage.completion_tokens',
+    'gen_ai.usage.output_tokens',
+    'llm.usage.completion_tokens',
+    'output_tokens',
+];
+const CACHE_READ_TOKEN_KEYS = [
+    'gen_ai.usage.cache_read_input_tokens',
+    'gen_ai.usage.cache_read_tokens',
+    'cache_read_tokens',
+];
+const CACHE_CREATION_TOKEN_KEYS = [
+    'gen_ai.usage.cache_creation_input_tokens',
+    'gen_ai.usage.cache_creation_tokens',
+    'cache_creation_tokens',
+];
+const MODEL_ID_KEYS = ['gen_ai.request.model', 'gen_ai.response.model', 'model'];
+const ALL_TOKEN_KEYS = [
+    ...PROMPT_TOKEN_KEYS,
+    ...COMPLETION_TOKEN_KEYS,
+    ...CACHE_READ_TOKEN_KEYS,
+    ...CACHE_CREATION_TOKEN_KEYS,
+];
 /** Build a TracesAccessor from a flat list of spans. */
 export function buildTracesAccessor(spans) {
     let totalTokens = 0;
-    let totalCost = 0;
+    let reportedCost = 0;
+    let computedCost = 0;
+    let hadReportedCost = false;
+    let hadComputedCost = false;
+    let tokenBearingSpanCount = 0;
+    // codex_review (PR #440): a span can have recognized tokens yet resolve
+    // NEITHER a reported cost NOR a computed one (unpriced/unresolved model) —
+    // this counter drives `costUnknowable` below so totalCost refuses to
+    // silently under-report just because *some other* span in the same trace
+    // happened to be priceable.
+    let unresolvedCostSpanCount = 0;
     const toolCalls = [];
     for (const span of spans) {
         const attrs = span.attributes ?? {};
-        const promptTokens = pickNumber(attrs, [
-            'gen_ai.usage.prompt_tokens',
-            'gen_ai.usage.input_tokens',
-            'llm.usage.prompt_tokens',
-        ]);
-        const completionTokens = pickNumber(attrs, [
-            'gen_ai.usage.completion_tokens',
-            'gen_ai.usage.output_tokens',
-            'llm.usage.completion_tokens',
-        ]);
+        const promptTokens = pickNumber(attrs, PROMPT_TOKEN_KEYS);
+        const completionTokens = pickNumber(attrs, COMPLETION_TOKEN_KEYS);
+        const cacheReadTokens = pickNumber(attrs, CACHE_READ_TOKEN_KEYS);
+        const cacheCreationTokens = pickNumber(attrs, CACHE_CREATION_TOKEN_KEYS);
+        const hasTokenAttr = promptTokens !== undefined ||
+            completionTokens !== undefined ||
+            cacheReadTokens !== undefined ||
+            cacheCreationTokens !== undefined;
+        if (hasTokenAttr)
+            tokenBearingSpanCount += 1;
         if (promptTokens)
             totalTokens += promptTokens;
         if (completionTokens)
             totalTokens += completionTokens;
-        const cost = pickNumber(attrs, [
+        if (cacheReadTokens)
+            totalTokens += cacheReadTokens;
+        if (cacheCreationTokens)
+            totalTokens += cacheCreationTokens;
+        const reported = pickNumber(attrs, [
             'gen_ai.usage.cost_usd',
             'gen_ai.usage.cost',
             'llm.usage.cost_usd',
         ]);
-        if (cost)
-            totalCost += cost;
+        if (reported !== undefined) {
+            hadReportedCost = true;
+            reportedCost += reported;
+        }
+        else if (hasTokenAttr) {
+            // No reported cost attribute on this span — fall back to a computed
+            // estimate from token counts, if we know pricing for the model.
+            const modelId = pickString(attrs, MODEL_ID_KEYS);
+            const rates = lookupModelRates(modelId);
+            if (rates) {
+                hadComputedCost = true;
+                computedCost += computeCost({
+                    input: promptTokens ?? 0,
+                    output: completionTokens ?? 0,
+                    cacheRead: cacheReadTokens ?? 0,
+                    cacheCreation: cacheCreationTokens ?? 0,
+                }, rates);
+            }
+            else {
+                // This span's tokens contribute to totalTokens but NOT to totalCost —
+                // without this counter that would be a silent, plausible-looking
+                // undercount rather than the loud failure #230 asks for.
+                unresolvedCostSpanCount += 1;
+            }
+        }
         // Tool spans
         const toolName = pickString(attrs, ['gen_ai.tool.name', 'llm.tool.name']);
         if (toolName) {
             toolCalls.push({ name: toolName, durationMs: spanDurationMs(span) });
         }
     }
+    const totalCostValue = reportedCost + computedCost;
+    const costSource = hadReportedCost && hadComputedCost
+        ? 'mixed'
+        : hadReportedCost
+            ? 'reported'
+            : hadComputedCost
+                ? 'computed'
+                : 'none';
+    // Issue class: spans exist, but none of them carry any token-usage
+    // attribute we recognize. Reading totalTokens/totalCost would silently
+    // return 0, letting matchers like `expect(traces.totalCost).lessThan(2)`
+    // pass vacuously (mirrors the #230 fail-loud philosophy). A trace that
+    // genuinely has recognized-but-zero-valued attributes is fine and must
+    // NOT throw — only the "we found nothing we understand" case throws.
+    const noRecognizedTokenAttrs = spans.length > 0 && tokenBearingSpanCount === 0;
+    // Issue class: tokens were found, but AT LEAST ONE token-bearing span
+    // resolved neither a reported cost nor a computed one (unpriced model,
+    // no cost attribute). Pre-codex_review this only threw when EVERY span
+    // was unresolved — a trace with 9 priced spans and 1 unpriced span
+    // silently under-reported totalCost by exactly the unpriced span's real
+    // spend while still returning a plausible-looking number (PR #440 review
+    // finding, blocker). Throw only when reading totalCost (tokens themselves
+    // are legitimately known in this case) — any unresolved span makes the
+    // total untrustworthy, not just a total absence of resolved spans.
+    const costUnknowable = totalTokens > 0 && unresolvedCostSpanCount > 0;
     return {
-        totalTokens,
-        totalCost,
+        get totalTokens() {
+            if (noRecognizedTokenAttrs) {
+                throw new Error(`traces fixture: ${spans.length} span(s) present but none carried a recognized ` +
+                    `token-usage attribute (checked ${ALL_TOKEN_KEYS.join(', ')}). Reading totalTokens ` +
+                    `would silently return 0 — either this agent's spans use an unsupported attribute ` +
+                    `shape (extend buildTracesAccessor() in lib/matchers/traces.ts), or this test case ` +
+                    `has no LLM spans and should use traces.spans / traces.toolCalls instead of totalTokens.`);
+            }
+            return totalTokens;
+        },
+        get totalCost() {
+            if (noRecognizedTokenAttrs && !hadReportedCost) {
+                throw new Error(`traces fixture: ${spans.length} span(s) present but none carried a recognized ` +
+                    `token-usage attribute or a cost attribute, so totalCost cannot be determined. ` +
+                    `Reading totalCost would silently return 0.`);
+            }
+            if (costUnknowable) {
+                throw new Error(`traces fixture: ${unresolvedCostSpanCount} of ${tokenBearingSpanCount} span(s) with token usage ` +
+                    `carried neither a cost attribute nor a model id matching the fallback pricing table (see ` +
+                    `lib/matchers/tracesPricing.ts). Reading totalCost would silently UNDER-report real spend by ` +
+                    `omitting those spans rather than reflect it -- add the model(s) to the pricing table or stamp a ` +
+                    `cost attribute on the span(s), or use traces.spans to inspect per-span usage directly.`);
+            }
+            return totalCostValue;
+        },
+        get costSource() {
+            return costSource;
+        },
         toolCalls,
         spanDuration(name) {
             const hit = spans.find(s => s.name === name);

package/lib/dist/lib/matchers/traces.js.map

@@ -1 +1 @@
-{"version":3,"file":"traces.js","sourceRoot":"","sources":["../../../matchers/traces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAqCH;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO;QACL,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,EAAE;QACb,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QACrB,KAAK,EAAE,EAAE;KACV,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,yBAAyB,CAAC,MAAc;IACtD,MAAM,IAAI,GAAG,GAAU,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC,CAAC;IACF,OAAO;QACL,IAAI,WAAW,KAAa,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5C,IAAI,SAAS,KAAa,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,IAAI,SAAS,KAA0D,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,KAA0B,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QACnD,YAAY,CAAC,KAAa,IAAY,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;KACrC,CAAC;AACtB,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,SAAS,GAA2C,EAAE,CAAC;IAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACpC,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,EAAE;YACrC,4BAA4B;YAC5B,2BAA2B;YAC3B,yBAAyB;SAC1B,CAAC,CAAC;QACH,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,EAAE;YACzC,gCAAgC;YAChC,4BAA4B;YAC5B,6BAA6B;SAC9B,CAAC,CAAC;QACH,IAAI,YAAY;YAAE,WAAW,IAAI,YAAY,CAAC;QAC9C,IAAI,gBAAgB;YAAE,WAAW,IAAI,gBAAgB,CAAC;QAEtD,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE;YAC7B,uBAAuB;YACvB,mBAAmB;YACnB,oBAAoB;SACrB,CAAC,CAAC;QACH,IAAI,IAAI;YAAE,SAAS,IAAI,IAAI,CAAC;QAE5B,aAAa;QACb,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,CAAC;QAC1E,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,OAAO;QACL,WAAW;QACX,SAAS;QACT,SAAS;QACT,YAAY,CAAC,IAAY;YACvB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAU;IAChC,MAAM,KAAK,GAAI,IAAY,CAAC,iBAAiB,IAAK,IAAY,CAAC,SAAS,CAAC;IACzE,MAAM,GAAG,GAAI,IAAY,CAAC,eAAe,IAAK,IAAY,CAAC,OAAO,CAAC;IACnE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACzD,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,OAAQ,IAAY,CAAC,UAAU,KAAK,QAAQ;QAAE,OAAQ,IAAY,CAAC,UAAU,CAAC;IAClF,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,UAAU,CAAC,KAA8B,EAAE,IAAc;IAChE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,UAAU,CAAC,KAA8B,EAAE,IAAc;IAChE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"traces.js","sourceRoot":"","sources":["../../../matchers/traces.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA+BnE;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO;QACL,WAAW,EAAE,CAAC;QACd,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,MAAM;QAClB,SAAS,EAAE,EAAE;QACb,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QACrB,KAAK,EAAE,EAAE;KACV,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,yBAAyB,CAAC,MAAc;IACtD,MAAM,IAAI,GAAG,GAAU,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;IAC3D,CAAC,CAAC;IACF,OAAO;QACL,IAAI,WAAW,KAAa,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC5C,IAAI,SAAS,KAAa,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,KAAiB,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,IAAI,SAAS,KAA0D,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,KAA0B,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QACnD,YAAY,CAAC,KAAa,IAAY,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;KACrC,CAAC;AACtB,CAAC;AAED,MAAM,iBAAiB,GAAG;IACxB,4BAA4B;IAC5B,2BAA2B;IAC3B,yBAAyB;IACzB,cAAc;CACf,CAAC;AACF,MAAM,qBAAqB,GAAG;IAC5B,gCAAgC;IAChC,4BAA4B;IAC5B,6BAA6B;IAC7B,eAAe;CAChB,CAAC;AACF,MAAM,qBAAqB,GAAG;IAC5B,sCAAsC;IACtC,gCAAgC;IAChC,mBAAmB;CACpB,CAAC;AACF,MAAM,yBAAyB,GAAG;IAChC,0CAA0C;IAC1C,oCAAoC;IACpC,uBAAuB;CACxB,CAAC;AACF,MAAM,aAAa,GAAG,CAAC,sBAAsB,EAAE,uBAAuB,EAAE,OAAO,CAAC,CAAC;AACjF,MAAM,cAAc,GAAG;IACrB,GAAG,iBAAiB;IACpB,GAAG,qBAAqB;IACxB,GAAG,qBAAqB;IACxB,GAAG,yBAAyB;CAC7B,CAAC;AAEF,wDAAwD;AACxD,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAC/C,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,wEAAwE;IACxE,2EAA2E;IAC3E,qEAAqE;IACrE,yEAAyE;IACzE,4BAA4B;IAC5B,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAChC,MAAM,SAAS,GAA2C,EAAE,CAAC;IAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACpC,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;QAC1D,MAAM,gBAAgB,GAAG,UAAU,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QAClE,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACjE,MAAM,mBAAmB,GAAG,UAAU,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;QAEzE,MAAM,YAAY,GAChB,YAAY,KAAK,SAAS;YAC1B,gBAAgB,KAAK,SAAS;YAC9B,eAAe,KAAK,SAAS;YAC7B,mBAAmB,KAAK,SAAS,CAAC;QAEpC,IAAI,YAAY;YAAE,qBAAqB,IAAI,CAAC,CAAC;QAC7C,IAAI,YAAY;YAAE,WAAW,IAAI,YAAY,CAAC;QAC9C,IAAI,gBAAgB;YAAE,WAAW,IAAI,gBAAgB,CAAC;QACtD,IAAI,eAAe;YAAE,WAAW,IAAI,eAAe,CAAC;QACpD,IAAI,mBAAmB;YAAE,WAAW,IAAI,mBAAmB,CAAC;QAE5D,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE;YACjC,uBAAuB;YACvB,mBAAmB;YACnB,oBAAoB;SACrB,CAAC,CAAC;QACH,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,eAAe,GAAG,IAAI,CAAC;YACvB,YAAY,IAAI,QAAQ,CAAC;QAC3B,CAAC;aAAM,IAAI,YAAY,EAAE,CAAC;YACxB,oEAAoE;YACpE,gEAAgE;YAChE,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACjD,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,eAAe,GAAG,IAAI,CAAC;gBACvB,YAAY,IAAI,WAAW,CACzB;oBACE,KAAK,EAAE,YAAY,IAAI,CAAC;oBACxB,MAAM,EAAE,gBAAgB,IAAI,CAAC;oBAC7B,SAAS,EAAE,eAAe,IAAI,CAAC;oBAC/B,aAAa,EAAE,mBAAmB,IAAI,CAAC;iBACxC,EACD,KAAK,CACN,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,iEAAiE;gBACjE,yDAAyD;gBACzD,uBAAuB,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,aAAa;QACb,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,kBAAkB,EAAE,eAAe,CAAC,CAAC,CAAC;QAC1E,IAAI,QAAQ,EAAE,CAAC;YACb,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAG,YAAY,GAAG,YAAY,CAAC;IACnD,MAAM,UAAU,GAAe,eAAe,IAAI,eAAe;QAC/D,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,eAAe;YACf,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,eAAe;gBACf,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,MAAM,CAAC;IAEf,mEAAmE;IACnE,uEAAuE;IACvE,yEAAyE;IACzE,uEAAuE;IACvE,uEAAuE;IACvE,qEAAqE;IACrE,MAAM,sBAAsB,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,qBAAqB,KAAK,CAAC,CAAC;IAE/E,sEAAsE;IACtE,uEAAuE;IACvE,uEAAuE;IACvE,mEAAmE;IACnE,wEAAwE;IACxE,yEAAyE;IACzE,0EAA0E;IAC1E,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,cAAc,GAAG,WAAW,GAAG,CAAC,IAAI,uBAAuB,GAAG,CAAC,CAAC;IAEtE,OAAO;QACL,IAAI,WAAW;YACb,IAAI,sBAAsB,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACb,mBAAmB,KAAK,CAAC,MAAM,iDAAiD;oBAChF,kCAAkC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB;oBACpF,mFAAmF;oBACnF,oFAAoF;oBACpF,yFAAyF,CAC1F,CAAC;YACJ,CAAC;YACD,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,IAAI,SAAS;YACX,IAAI,sBAAsB,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CACb,mBAAmB,KAAK,CAAC,MAAM,iDAAiD;oBAChF,gFAAgF;oBAChF,4CAA4C,CAC7C,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CACb,mBAAmB,uBAAuB,OAAO,qBAAqB,4BAA4B;oBAClG,2FAA2F;oBAC3F,8FAA8F;oBAC9F,kGAAkG;oBAClG,wFAAwF,CACzF,CAAC;YACJ,CAAC;YACD,OAAO,cAAc,CAAC;QACxB,CAAC;QACD,IAAI,UAAU;YACZ,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,SAAS;QACT,YAAY,CAAC,IAAY;YACvB,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAC7C,OAAO,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAU;IAChC,MAAM,KAAK,GAAI,IAAY,CAAC,iBAAiB,IAAK,IAAY,CAAC,SAAS,CAAC;IACzE,MAAM,GAAG,GAAI,IAAY,CAAC,eAAe,IAAK,IAAY,CAAC,OAAO,CAAC;IACnE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACzD,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,OAAQ,IAAY,CAAC,UAAU,KAAK,QAAQ;QAAE,OAAQ,IAAY,CAAC,UAAU,CAAC;IAClF,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,UAAU,CAAC,KAA8B,EAAE,IAAc;IAChE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1D,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,UAAU,CAAC,KAA8B,EAAE,IAAc;IAChE,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/lib/dist/lib/matchers/tracesPricing.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Fallback pricing table used by `buildTracesAccessor()` (see
+ * `lib/matchers/traces.ts`) to *compute* `totalCost` when spans carry
+ * token-usage attributes but no cost attribute (this is the common case —
+ * most OTel exporters, including Claude Code's, emit raw token counts and
+ * leave cost calculation to the consumer).
+ *
+ * Rates are USD per **million tokens** (MTok), converted to per-token below.
+ * These are point-in-time *estimates* maintained by hand — update them when
+ * pricing changes. They are intentionally separate from
+ * `server/services/codingAgents/pricing.ts` (used by the coding-agent
+ * analytics/leaderboard feature): that table silently falls back to Sonnet
+ * pricing for unknown models, which is the wrong behavior here — an eval
+ * reading `traces.totalCost` for an unpriced model must know the number is
+ * incomplete (see `buildTracesAccessor`'s fail-loud handling), not get a
+ * silently-substituted estimate.
+ */
+export interface ModelRates {
+    /** USD per input (prompt) token. */
+    input: number;
+    /** USD per output (completion) token. */
+    output: number;
+    /** USD per cache-read token (cheaper than a fresh input token). */
+    cacheRead: number;
+    /** USD per cache-creation/write token (pricier than a fresh input token). */
+    cacheCreation: number;
+}
+/** Look up rates for a model id. Returns `undefined` when no rule matches. */
+export declare function lookupModelRates(modelId: string | undefined): ModelRates | undefined;
+export interface UsageTokens {
+    input: number;
+    output: number;
+    cacheRead: number;
+    cacheCreation: number;
+}
+/** Compute USD cost for a set of token counts against a model's rates. */
+export declare function computeCost(tokens: UsageTokens, rates: ModelRates): number;
+//# sourceMappingURL=tracesPricing.d.ts.map

package/lib/dist/lib/matchers/tracesPricing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tracesPricing.d.ts","sourceRoot":"","sources":["../../../matchers/tracesPricing.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,yCAAyC;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,aAAa,EAAE,MAAM,CAAC;CACvB;AA8CD,8EAA8E;AAC9E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAMpF;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,0EAA0E;AAC1E,wBAAgB,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,GAAG,MAAM,CAO1E"}

package/lib/dist/lib/matchers/tracesPricing.js

@@ -0,0 +1,64 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+const PER_MTOK = 1_000_000;
+/**
+ * Estimated Bedrock/Anthropic Claude rates, USD per MTok:
+ *   Sonnet class: $3 in / $15 out / $0.30 cache-read / $3.75 cache-write
+ *   Opus class:   $15 in / $75 out / $1.50 cache-read / $18.75 cache-write
+ *   Haiku class:  $0.80 in / $4 out / $0.08 cache-read / $1.00 cache-write
+ * Keyed by substrings matched case-insensitively against the span's model
+ * id (see `resolveModelId`), so both raw Anthropic ids
+ * (`claude-sonnet-4-6-...`) and Bedrock ids
+ * (`us.anthropic.claude-sonnet-4-6-...`) match.
+ */
+const RATE_TABLE = [
+    {
+        match: /opus/i,
+        rates: {
+            input: 15 / PER_MTOK,
+            output: 75 / PER_MTOK,
+            cacheRead: 1.5 / PER_MTOK,
+            cacheCreation: 18.75 / PER_MTOK,
+        },
+    },
+    {
+        match: /haiku/i,
+        rates: {
+            input: 0.8 / PER_MTOK,
+            output: 4 / PER_MTOK,
+            cacheRead: 0.08 / PER_MTOK,
+            cacheCreation: 1.0 / PER_MTOK,
+        },
+    },
+    {
+        // Sonnet is the default Claude class; matched last so opus/haiku take
+        // priority, but also serves as the generic "claude" match.
+        match: /sonnet|claude/i,
+        rates: {
+            input: 3 / PER_MTOK,
+            output: 15 / PER_MTOK,
+            cacheRead: 0.3 / PER_MTOK,
+            cacheCreation: 3.75 / PER_MTOK,
+        },
+    },
+];
+/** Look up rates for a model id. Returns `undefined` when no rule matches. */
+export function lookupModelRates(modelId) {
+    if (!modelId)
+        return undefined;
+    for (const { match, rates } of RATE_TABLE) {
+        if (match.test(modelId))
+            return rates;
+    }
+    return undefined;
+}
+/** Compute USD cost for a set of token counts against a model's rates. */
+export function computeCost(tokens, rates) {
+    return (tokens.input * rates.input +
+        tokens.output * rates.output +
+        tokens.cacheRead * rates.cacheRead +
+        tokens.cacheCreation * rates.cacheCreation);
+}
+//# sourceMappingURL=tracesPricing.js.map

package/lib/dist/lib/matchers/tracesPricing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tracesPricing.js","sourceRoot":"","sources":["../../../matchers/tracesPricing.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA+BH,MAAM,QAAQ,GAAG,SAAS,CAAC;AAE3B;;;;;;;;;GASG;AACH,MAAM,UAAU,GAAgD;IAC9D;QACE,KAAK,EAAE,OAAO;QACd,KAAK,EAAE;YACL,KAAK,EAAE,EAAE,GAAG,QAAQ;YACpB,MAAM,EAAE,EAAE,GAAG,QAAQ;YACrB,SAAS,EAAE,GAAG,GAAG,QAAQ;YACzB,aAAa,EAAE,KAAK,GAAG,QAAQ;SAChC;KACF;IACD;QACE,KAAK,EAAE,QAAQ;QACf,KAAK,EAAE;YACL,KAAK,EAAE,GAAG,GAAG,QAAQ;YACrB,MAAM,EAAE,CAAC,GAAG,QAAQ;YACpB,SAAS,EAAE,IAAI,GAAG,QAAQ;YAC1B,aAAa,EAAE,GAAG,GAAG,QAAQ;SAC9B;KACF;IACD;QACE,sEAAsE;QACtE,2DAA2D;QAC3D,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE;YACL,KAAK,EAAE,CAAC,GAAG,QAAQ;YACnB,MAAM,EAAE,EAAE,GAAG,QAAQ;YACrB,SAAS,EAAE,GAAG,GAAG,QAAQ;YACzB,aAAa,EAAE,IAAI,GAAG,QAAQ;SAC/B;KACF;CACF,CAAC;AAEF,8EAA8E;AAC9E,MAAM,UAAU,gBAAgB,CAAC,OAA2B;IAC1D,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC/B,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,KAAK,CAAC;IACxC,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AASD,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,MAAmB,EAAE,KAAiB;IAChE,OAAO,CACL,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;QAC1B,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM;QAC5B,MAAM,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS;QAClC,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAC3C,CAAC;AACJ,CAAC"}

package/lib/dist/lib/matchers/types.d.ts

@@ -71,5 +71,30 @@ export interface MatcherResult {
         trajectory_alignment_score?: number;
         [k: string]: number | undefined;
     };
+    /**
+     * True for a synthetic entry the runner appends when the test body threw
+     * before reaching further matcher calls in source order — distinct from
+     * `pass: false` (a matcher that DID run and failed). This row was never
+     * *attempted*; it marks that later expect()/judge()/evaluate() calls (if
+     * any) never ran because chai's bail-on-first-failure semantics stopped
+     * the body at the first throw. Always excluded from gate/pass-rate
+     * aggregation. See `appendNotReachedMarker()` in services/evaluation/
+     * index.ts and `expect.soft()` in lib/matchers/expect.ts for the mode
+     * that avoids needing this marker by not bailing at all.
+     */
+    notReached?: boolean;
+    /**
+     * Structured, non-metric judge output beyond the typed wire fields — the
+     * SDK-side mirror of `JudgeResponse.extraFields`. Any JSON key a judge
+     * prompt emits beyond the known schema lands here (captured by
+     * `server/services/judgeResponseParser.ts`), so prompt iteration surfaces
+     * new structure without code changes. Conventional keys the UI knows how
+     * to render when present:
+     *
+     *   - `facts`: Array<{ fact: string; verdict: 'stated'|'partial'|'missing'|'contradicted'; rationale?: string; credit?: number }>
+     *   - `failure_causes`: Array<{ cause: string; detail?: string; dimension?: string }>
+     *   - `evidence`: { expected_sources?: string[]; cited_sources?: Array<string | { id: string; title?: string }> }
+     */
+    judgeExtraFields?: Record<string, unknown>;
 }
 //# sourceMappingURL=types.d.ts.map

package/lib/dist/lib/matchers/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../matchers/types.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,aAAa,GACrB,gBAAgB,GAChB,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,qCAAqC;IACrC,MAAM,EAAE,aAAa,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IASf;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;KACrC,CAAC,CAAC;IAEH;;;;;OAKG;IACH,YAAY,CAAC,EAAE;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;KACjC,CAAC;CACH"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../matchers/types.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,aAAa,GACrB,gBAAgB,GAChB,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,qCAAqC;IACrC,MAAM,EAAE,aAAa,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IASf;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;KACrC,CAAC,CAAC;IAEH;;;;;OAKG;IACH,YAAY,CAAC,EAAE;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;KACjC,CAAC;IAEF;;;;;;;;;;OAUG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C"}

package/lib/dist/lib/resolveCanonicalRun.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Resolve the CANONICAL run object for an id that may exist in two shapes:
+ * a legacy `BenchmarkRun` projection embedded in `benchmark.runs[]` (never
+ * carries `docType`, never kept in sync after the initial write) and a
+ * first-class `EvaluationRun` doc (`docType: 'evaluation-run'`). Runs
+ * created WITH a benchmarkId are dual-written as both (see
+ * `server/routes/storage/evaluationRuns.ts`) -- the first-class doc is
+ * always the freshest/most capable representation when it exists.
+ *
+ * Extracted out of RunInspectorPage.tsx (the first caller) so the
+ * resolution logic is reusable and independently testable rather than a
+ * page-local pattern other components/route handlers would have to
+ * reinvent (see #462's Retry-judgement work, which needs the exact same
+ * resolution to make its own EvaluationRun-only capability checks
+ * meaningful on the benchmark-scoped route).
+ *
+ * `fetchEvaluationRun` is injected (rather than importing
+ * `services/client` directly) so this stays a plain, synchronously
+ * testable function with no module-mocking required.
+ */
+import type { BenchmarkRun, EvaluationRun } from '../types/index.js';
+export declare function resolveCanonicalEvaluationRun(runId: string, embeddedProjection: BenchmarkRun, fetchEvaluationRun: (id: string) => Promise<EvaluationRun>): Promise<BenchmarkRun | EvaluationRun>;
+//# sourceMappingURL=resolveCanonicalRun.d.ts.map

package/lib/dist/lib/resolveCanonicalRun.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveCanonicalRun.d.ts","sourceRoot":"","sources":["../../resolveCanonicalRun.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEpE,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,MAAM,EACb,kBAAkB,EAAE,YAAY,EAChC,kBAAkB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,GACzD,OAAO,CAAC,YAAY,GAAG,aAAa,CAAC,CAsBvC"}

package/lib/dist/lib/resolveCanonicalRun.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export async function resolveCanonicalEvaluationRun(runId, embeddedProjection, fetchEvaluationRun) {
+    try {
+        // Defensive `?? embeddedProjection`: some test doubles / API layers
+        // resolve to a falsy value on "not found" instead of throwing.
+        return (await fetchEvaluationRun(runId)) ?? embeddedProjection;
+    }
+    catch (err) {
+        // A 404 means this run only ever exists as a legacy BenchmarkRun
+        // (pre-#399, no first-class doc) -- expected, silent fallback. Any
+        // OTHER failure (500, network error, auth) must NOT be silently
+        // treated the same way: falling back is still the right availability
+        // choice for a read-only inspector page (this page already degrades
+        // gracefully elsewhere -- see loadData()'s report-summary fallback),
+        // but masking a real failure identically to "doesn't exist" would
+        // hide it from anyone debugging why results/stats look stale.
+        if (err?.status !== 404) {
+            console.warn(`[resolveCanonicalEvaluationRun] Failed to fetch first-class EvaluationRun doc for ${runId} (falling back to the embedded projection):`, err?.message ?? err);
+        }
+        return embeddedProjection;
+    }
+}
+//# sourceMappingURL=resolveCanonicalRun.js.map

package/lib/dist/lib/resolveCanonicalRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveCanonicalRun.js","sourceRoot":"","sources":["../../resolveCanonicalRun.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyBH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,KAAa,EACb,kBAAgC,EAChC,kBAA0D;IAE1D,IAAI,CAAC;QACH,oEAAoE;QACpE,+DAA+D;QAC/D,OAAO,CAAC,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC,IAAI,kBAAkB,CAAC;IACjE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,iEAAiE;QACjE,mEAAmE;QACnE,gEAAgE;QAChE,qEAAqE;QACrE,oEAAoE;QACpE,qEAAqE;QACrE,kEAAkE;QAClE,8DAA8D;QAC9D,IAAI,GAAG,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CACV,qFAAqF,KAAK,6CAA6C,EACvI,GAAG,EAAE,OAAO,IAAI,GAAG,CACpB,CAAC;QACJ,CAAC;QACD,OAAO,kBAAkB,CAAC;IAC5B,CAAC;AACH,CAAC"}