@opensearch-project/agent-health 0.6.0 → 0.7.0

package/lib/dist/lib/matchers/judgeReasoningParse.js

@@ -0,0 +1,128 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/** Map a matched verdict keyword onto the canonical kind. */
+function verdictKind(keyword) {
+    const k = keyword.toUpperCase();
+    if (k.includes('CONTRADICT') && !k.includes('MISSING'))
+        return 'contradicted';
+    if (k.includes('MISSING') || k.includes('NOT STATED'))
+        return 'missing';
+    if (k.includes('PARTIAL'))
+        return 'partial';
+    return 'stated';
+}
+// Verdict keywords the judge writes inline after a fact. Order matters:
+// longest / most specific first so "MISSING/CONTRADICTED" isn't split.
+const VERDICT_KEYWORD = /(MISSING\s*\/\s*CONTRADICTED|FULLY\s+STATED|PARTIALLY\s+STATED|NOT\s+STATED|CONTRADICTED|MISSING|PARTIAL(?:LY)?)/i;
+// Parse at most this much reasoning — real verdicts are a few KB; a hard cap
+// bounds regex work on pathological inputs (defense against backtracking
+// blowups on adversarial multi-hundred-KB strings).
+const MAX_PARSE_CHARS = 20_000;
+// A fact item: list marker (`1.`, `1)`, `**Fact 1:`, `Required fact 1 (`),
+// then the fact text (quoted or plain), a separator (—, -, :, en-dash, `):`),
+// then the verdict keyword. Fact text is capped to keep matches sane.
+// NOTE: deliberately no lookbehind — constructed-at-import regexes with
+// lookbehind hard-crash report rendering on older WebKit. The leading
+// whitespace/newline is consumed instead (harmless: markers never overlap).
+const FACT_ITEM = new RegExp(String.raw `(?:^|[\s\n])` + // start of string or after whitespace (inline numbered lists)
+    String.raw `(?:\*\*)?(?:Required\s+fact\s+\d+|Fact\s+\d+|\d+)\s*[.):\u2013\u2014-]?\s*` + // marker
+    String.raw `(?:\*\*)?\s*` +
+    String.raw `['‘"“(]?(.{4,240}?)['’"”)]?` + // fact text (lazy)
+    String.raw `(?:\*\*)?\s*[\u2013\u2014:(\u2015-]+\s*(?:\*\*)?\s*` + // separator
+    VERDICT_KEYWORD.source + // verdict
+    String.raw `(?:\s+stated)?` + // "PARTIALLY stated"
+    String.raw `[.!]?\s*` +
+    // Trailing note: lazy, stopped by the NEXT numbered/bold fact item on the
+    // same line (inline lists put every fact in one paragraph) or line end.
+    String.raw `([^\n]*?)(?=\s\d+[.)]\s*['‘"“(*]|\s\*\*(?:Required\s+fact|Fact)|\n|$)`, 'gi');
+/**
+ * Extract per-required-fact verdicts from judge reasoning prose.
+ * Returns `[]` when nothing that looks like a fact list is present —
+ * callers must fall back to showing the raw reasoning.
+ */
+export function parseFactVerdicts(reasoning) {
+    if (!reasoning || reasoning.length < 20)
+        return [];
+    const text = reasoning.slice(0, MAX_PARSE_CHARS);
+    const out = [];
+    const seen = new Set();
+    for (const m of text.matchAll(FACT_ITEM)) {
+        const factRaw = (m[1] ?? '').trim();
+        const keyword = m[2] ?? '';
+        // Guard against summary-phrase false positives like
+        // "4 facts fully stated (1.0 each)": require real fact text, not a
+        // recap that itself talks about facts/statements in aggregate.
+        if (!factRaw || factRaw.length < 8)
+            continue;
+        if (/^facts?\b/i.test(factRaw) || /\bfacts fully stated\b/i.test(factRaw))
+            continue;
+        // Strip markdown/bold leftovers and trailing separators.
+        const fact = factRaw.replace(/\*\*/g, '').replace(/[\u2013\u2014:-]+$/, '').trim();
+        const key = fact.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        let note = (m[3] ?? '').replace(/\*\*/g, '').trim().slice(0, 220);
+        // Notes that are just the start of the next list item are noise.
+        if (/^\d+[.)]/.test(note))
+            note = '';
+        out.push({ fact, verdict: verdictKind(keyword), ...(note ? { note } : {}) });
+        if (out.length >= 12)
+            break; // sanity cap
+    }
+    return out;
+}
+// Hex-ish document/article ids (8+ chars, at least one a–f so bare integers
+// like "10000000" never qualify) — what RAG corpora and judge reasoning use
+// when naming expected vs cited sources.
+const HEX_ID_SCAN = /\b(?=[0-9]*[a-f])[0-9a-f]{8,64}\b/i;
+/** First hex id within `window` chars after `index` in `text`, if any. */
+function firstIdAfter(text, index, window = 140) {
+    const m = text.slice(index, index + window).match(HEX_ID_SCAN);
+    return m?.[0];
+}
+/**
+ * Detect an "expected source X but cited/retrieved Y" statement.
+ * Conservative: both ids must be present and differ.
+ */
+export function parseSourceMismatch(reasoning) {
+    if (!reasoning)
+        return null;
+    const text = reasoning.slice(0, MAX_PARSE_CHARS);
+    // Expected id: first hex id shortly after an "expected source …" mention.
+    // (A character-class "gap" can't work here — prose like "document is
+    // article" contains hex letters — so scan a window for the first id.)
+    const expectedKw = text.match(/expected\s+source\s+(?:document|article)?/i);
+    if (!expectedKw || expectedKw.index === undefined)
+        return null;
+    const expected = firstIdAfter(text, expectedKw.index + expectedKw[0].length);
+    if (!expected)
+        return null;
+    // Cited id: first differing hex id shortly after a cite/retrieve verb.
+    const citedKw = /(?:\bcited\b|\bretrieved\b|\busing\s+article\b|\bcites\b)/gi;
+    for (const m of text.matchAll(citedKw)) {
+        if (m.index === undefined)
+            continue;
+        const id = firstIdAfter(text, m.index + m[0].length);
+        if (id && id.toLowerCase() !== expected.toLowerCase()) {
+            return { expected, cited: id };
+        }
+    }
+    // Compact form: "(b6c9353c vs 49d9e88f)" — either order relative to expected.
+    const vs = text.match(new RegExp(String.raw `([0-9a-f]{8,64})\s*(?:vs\.?|versus)\s*([0-9a-f]{8,64})`, 'i'));
+    if (vs) {
+        const [a, b] = [vs[1], vs[2]];
+        const other = a.toLowerCase() === expected.toLowerCase() ? b : a;
+        if (other && other.toLowerCase() !== expected.toLowerCase()) {
+            return { expected, cited: other };
+        }
+    }
+    return null;
+}
+/** Shorten a long doc id for display: `49d9e88fadbf…` (first 8 chars). */
+export function shortId(id) {
+    return id.length > 12 ? `${id.slice(0, 8)}…` : id;
+}
+//# sourceMappingURL=judgeReasoningParse.js.map

package/lib/dist/lib/matchers/judgeReasoningParse.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judgeReasoningParse.js","sourceRoot":"","sources":["../../../matchers/judgeReasoningParse.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyCH,6DAA6D;AAC7D,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAChC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,cAAc,CAAC;IAC9E,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,wEAAwE;AACxE,uEAAuE;AACvE,MAAM,eAAe,GACnB,mHAAmH,CAAC;AAEtH,6EAA6E;AAC7E,yEAAyE;AACzE,oDAAoD;AACpD,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,2EAA2E;AAC3E,8EAA8E;AAC9E,sEAAsE;AACtE,wEAAwE;AACxE,sEAAsE;AACtE,4EAA4E;AAC5E,MAAM,SAAS,GAAG,IAAI,MAAM,CAC1B,MAAM,CAAC,GAAG,CAAA,cAAc,GAAG,8DAA8D;IACvF,MAAM,CAAC,GAAG,CAAA,4EAA4E,GAAG,SAAS;IAClG,MAAM,CAAC,GAAG,CAAA,cAAc;IACxB,MAAM,CAAC,GAAG,CAAA,6BAA6B,GAAG,mBAAmB;IAC7D,MAAM,CAAC,GAAG,CAAA,qDAAqD,GAAG,YAAY;IAC9E,eAAe,CAAC,MAAM,GAAG,UAAU;IACnC,MAAM,CAAC,GAAG,CAAA,gBAAgB,GAAG,qBAAqB;IAClD,MAAM,CAAC,GAAG,CAAA,UAAU;IACpB,0EAA0E;IAC1E,wEAAwE;IACxE,MAAM,CAAC,GAAG,CAAA,uEAAuE,EACnF,IAAI,CACL,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAA6B;IAC7D,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IACjD,MAAM,GAAG,GAAwB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,oDAAoD;QACpD,mEAAmE;QACnE,+DAA+D;QAC/D,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC7C,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,SAAS;QACpF,yDAAyD;QACzD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACnF,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAClE,iEAAiE;QACjE,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,IAAI,GAAG,EAAE,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC7E,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE;YAAE,MAAM,CAAC,aAAa;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4EAA4E;AAC5E,4EAA4E;AAC5E,yCAAyC;AACzC,MAAM,WAAW,GAAG,oCAAoC,CAAC;AAEzD,0EAA0E;AAC1E,SAAS,YAAY,CAAC,IAAY,EAAE,KAAa,EAAE,MAAM,GAAG,GAAG;IAC7D,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/D,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAA6B;IAC/D,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IAEjD,0EAA0E;IAC1E,qEAAqE;IACrE,sEAAsE;IACtE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAC5E,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/D,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC7E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,uEAAuE;IACvE,MAAM,OAAO,GAAG,6DAA6D,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;YAAE,SAAS;QACpC,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YACtD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CACnB,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAA,wDAAwD,EAAE,GAAG,CAAC,CACpF,CAAC;IACF,IAAI,EAAE,EAAE,CAAC;QACP,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,IAAI,KAAK,IAAI,KAAK,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;YAC5D,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,OAAO,CAAC,EAAU;IAChC,OAAO,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AACpD,CAAC"}

package/lib/dist/lib/matchers/types.d.ts

@@ -71,5 +71,30 @@ export interface MatcherResult {
         trajectory_alignment_score?: number;
         [k: string]: number | undefined;
     };
+    /**
+     * True for a synthetic entry the runner appends when the test body threw
+     * before reaching further matcher calls in source order — distinct from
+     * `pass: false` (a matcher that DID run and failed). This row was never
+     * *attempted*; it marks that later expect()/judge()/evaluate() calls (if
+     * any) never ran because chai's bail-on-first-failure semantics stopped
+     * the body at the first throw. Always excluded from gate/pass-rate
+     * aggregation. See `appendNotReachedMarker()` in services/evaluation/
+     * index.ts and `expect.soft()` in lib/matchers/expect.ts for the mode
+     * that avoids needing this marker by not bailing at all.
+     */
+    notReached?: boolean;
+    /**
+     * Structured, non-metric judge output beyond the typed wire fields — the
+     * SDK-side mirror of `JudgeResponse.extraFields`. Any JSON key a judge
+     * prompt emits beyond the known schema lands here (captured by
+     * `server/services/judgeResponseParser.ts`), so prompt iteration surfaces
+     * new structure without code changes. Conventional keys the UI knows how
+     * to render when present:
+     *
+     *   - `facts`: Array<{ fact: string; verdict: 'stated'|'partial'|'missing'|'contradicted'; rationale?: string; credit?: number }>
+     *   - `failure_causes`: Array<{ cause: string; detail?: string; dimension?: string }>
+     *   - `evidence`: { expected_sources?: string[]; cited_sources?: Array<string | { id: string; title?: string }> }
+     */
+    judgeExtraFields?: Record<string, unknown>;
 }
 //# sourceMappingURL=types.d.ts.map

package/lib/dist/lib/matchers/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../matchers/types.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,aAAa,GACrB,gBAAgB,GAChB,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,qCAAqC;IACrC,MAAM,EAAE,aAAa,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IASf;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;KACrC,CAAC,CAAC;IAEH;;;;;OAKG;IACH,YAAY,CAAC,EAAE;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;KACjC,CAAC;CACH"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../matchers/types.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,aAAa,GACrB,gBAAgB,GAChB,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,aAAa;IAC5B,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,mCAAmC;IACnC,IAAI,EAAE,OAAO,CAAC;IACd,qCAAqC;IACrC,MAAM,EAAE,aAAa,CAAC;IACtB;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,kEAAkE;IAClE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IAGtB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAC;IASf;;;;;OAKG;IACH,qBAAqB,CAAC,EAAE,KAAK,CAAC;QAC5B,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;KACrC,CAAC,CAAC;IAEH;;;;;OAKG;IACH,YAAY,CAAC,EAAE;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,0BAA0B,CAAC,EAAE,MAAM,CAAC;QACpC,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;KACjC,CAAC;IAEF;;;;;;;;;;OAUG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;;;;;;;;OAWG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC5C"}

package/lib/dist/lib/resolveCanonicalRun.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Resolve the CANONICAL run object for an id that may exist in two shapes:
+ * a legacy `BenchmarkRun` projection embedded in `benchmark.runs[]` (never
+ * carries `docType`, never kept in sync after the initial write) and a
+ * first-class `EvaluationRun` doc (`docType: 'evaluation-run'`). Runs
+ * created WITH a benchmarkId are dual-written as both (see
+ * `server/routes/storage/evaluationRuns.ts`) -- the first-class doc is
+ * always the freshest/most capable representation when it exists.
+ *
+ * Extracted out of RunInspectorPage.tsx (the first caller) so the
+ * resolution logic is reusable and independently testable rather than a
+ * page-local pattern other components/route handlers would have to
+ * reinvent (see #462's Retry-judgement work, which needs the exact same
+ * resolution to make its own EvaluationRun-only capability checks
+ * meaningful on the benchmark-scoped route).
+ *
+ * `fetchEvaluationRun` is injected (rather than importing
+ * `services/client` directly) so this stays a plain, synchronously
+ * testable function with no module-mocking required.
+ */
+import type { BenchmarkRun, EvaluationRun } from '../types/index.js';
+export declare function resolveCanonicalEvaluationRun(runId: string, embeddedProjection: BenchmarkRun, fetchEvaluationRun: (id: string) => Promise<EvaluationRun>): Promise<BenchmarkRun | EvaluationRun>;
+//# sourceMappingURL=resolveCanonicalRun.d.ts.map

package/lib/dist/lib/resolveCanonicalRun.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveCanonicalRun.d.ts","sourceRoot":"","sources":["../../resolveCanonicalRun.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEpE,wBAAsB,6BAA6B,CACjD,KAAK,EAAE,MAAM,EACb,kBAAkB,EAAE,YAAY,EAChC,kBAAkB,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,CAAC,GACzD,OAAO,CAAC,YAAY,GAAG,aAAa,CAAC,CAsBvC"}

package/lib/dist/lib/resolveCanonicalRun.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export async function resolveCanonicalEvaluationRun(runId, embeddedProjection, fetchEvaluationRun) {
+    try {
+        // Defensive `?? embeddedProjection`: some test doubles / API layers
+        // resolve to a falsy value on "not found" instead of throwing.
+        return (await fetchEvaluationRun(runId)) ?? embeddedProjection;
+    }
+    catch (err) {
+        // A 404 means this run only ever exists as a legacy BenchmarkRun
+        // (pre-#399, no first-class doc) -- expected, silent fallback. Any
+        // OTHER failure (500, network error, auth) must NOT be silently
+        // treated the same way: falling back is still the right availability
+        // choice for a read-only inspector page (this page already degrades
+        // gracefully elsewhere -- see loadData()'s report-summary fallback),
+        // but masking a real failure identically to "doesn't exist" would
+        // hide it from anyone debugging why results/stats look stale.
+        if (err?.status !== 404) {
+            console.warn(`[resolveCanonicalEvaluationRun] Failed to fetch first-class EvaluationRun doc for ${runId} (falling back to the embedded projection):`, err?.message ?? err);
+        }
+        return embeddedProjection;
+    }
+}
+//# sourceMappingURL=resolveCanonicalRun.js.map

package/lib/dist/lib/resolveCanonicalRun.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveCanonicalRun.js","sourceRoot":"","sources":["../../resolveCanonicalRun.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAyBH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,KAAa,EACb,kBAAgC,EAChC,kBAA0D;IAE1D,IAAI,CAAC;QACH,oEAAoE;QACpE,+DAA+D;QAC/D,OAAO,CAAC,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC,IAAI,kBAAkB,CAAC;IACjE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,iEAAiE;QACjE,mEAAmE;QACnE,gEAAgE;QAChE,qEAAqE;QACrE,oEAAoE;QACpE,qEAAqE;QACrE,kEAAkE;QAClE,8DAA8D;QAC9D,IAAI,GAAG,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CACV,qFAAqF,KAAK,6CAA6C,EACvI,GAAG,EAAE,OAAO,IAAI,GAAG,CACpB,CAAC;QACJ,CAAC;QACD,OAAO,kBAAkB,CAAC;IAC5B,CAAC;AACH,CAAC"}

package/lib/dist/lib/runActions.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Pure, isomorphic predicates for the run-lifecycle action matrix (delete /
+ * cancel / re-run / retry-judgement) shared by every run surface (runs list,
+ * benchmark runs list, run detail/report page, inspector header) AND by the
+ * server routes that enforce the same rules server-side. No storage/IO here
+ * — callers pass in the run document they already have.
+ *
+ * Action matrix (see AGENTS.md / PR description for the full writeup):
+ *   - Delete: any run, any status. Always available (existing endpoint).
+ *   - Cancel: only while `status === 'running'`.
+ *   - Re-run: only top-level EvaluationRun docs (docType === 'evaluation-run').
+ *     Legacy benchmark-embedded BenchmarkRun rows don't support the
+ *     provenance-tracked rerun endpoint (pre-existing constraint — see
+ *     RunConfigDialog / EvalRunsPage).
+ *   - Retry judgement: only EvaluationRun docs, only when the run is
+ *     terminal (not running) AND it has at least one test case whose agent
+ *     execution completed but the judge produced NO verdict (a judge-failed
+ *     / "errored" case — trace timeout, judge 400, "evaluator could not
+ *     run" — as opposed to an agent-failed one — retrying the judge on a
+ *     case the agent itself never finished has nothing to re-grade). Same
+ *     predicate the retry-judgement pipeline itself selects on
+ *     (services/evaluation/retryJudgement.ts `isJudgeFailedCase`, keyed on
+ *     the report's `metricsStatus: 'error'`, which the runner mirrors onto
+ *     the run's results map as a `completed` result with no
+ *     `passFailStatus`) and that `lib/runStats` buckets as `errored`.
+ */
+import type { BenchmarkRun, EvaluationRun } from '../types/index.js';
+/** Minimal shape both BenchmarkRun and EvaluationRun satisfy for these checks. */
+export type RunLike = Pick<BenchmarkRun | EvaluationRun, 'status' | 'results'> & {
+    docType?: string;
+};
+/**
+ * True when `run` is a top-level EvaluationRun document (created via
+ * `POST /api/storage/evaluation-runs`), as opposed to a legacy
+ * benchmark-embedded BenchmarkRun (`benchmark.runs[]`). The two share a lot
+ * of shape but only EvaluationRun docs carry `docType: 'evaluation-run'` and
+ * support the rerun/retry-judgement endpoints.
+ *
+ * Null-tolerant wrapper over the typed predicate in `types/index.ts` (the
+ * single source of truth for the docType discriminator) — kept so callers
+ * holding a possibly-null run don't need their own guard.
+ */
+export declare function isEvaluationRun(run: RunLike | null | undefined): run is EvaluationRun;
+/** True while the run has an in-progress executor that a Cancel action could stop. */
+export declare function isRunRunning(run: RunLike | null | undefined): boolean;
+/** True once a run has reached any terminal state (not running/pending). */
+export declare function isRunTerminal(run: RunLike | null | undefined): boolean;
+/**
+ * Count test cases where the AGENT finished (`status === 'completed'`) but
+ * the JUDGE produced no verdict (`passFailStatus` neither 'passed' nor
+ * 'failed') — the "errored" bucket of `lib/runStats` `bucketRunResults`
+ * (issue #242) and exactly the set `POST .../retry-judgement` (default
+ * `scope=errored`) will re-judge. Deliberately excludes:
+ *   - `status !== 'completed'` (agent-failed/cancelled/pending cases — no
+ *     trajectory to re-judge, or nothing ran).
+ *   - a real 'failed' verdict — the judge DID run and graded the case; that
+ *     is a legitimate result, not a judge failure (re-grading it is
+ *     `scope=all`, opt-in from the inspector's dedicated button).
+ *
+ * `passFailStatus` isn't declared on `EvaluationRun['results']`'s static
+ * type (a pre-existing gap — evaluationRunner.ts writes it via an `as any`
+ * spread) so this reads it defensively.
+ */
+export declare function countJudgeFailed(run: RunLike | null | undefined): number;
+export interface RunActionVisibility {
+    /** Delete is always available for any run in any status. */
+    canDelete: boolean;
+    /** Cancel is available only while the run is actively running. */
+    canCancel: boolean;
+    /** Re-run is available only for top-level EvaluationRun docs. */
+    canRerun: boolean;
+    /** Reason to show (e.g. as a disabled-item tooltip) when canRerun is false. */
+    rerunDisabledReason?: string;
+    /** Retry judgement: EvaluationRun, terminal, with >0 judge-failed cases. */
+    canRetryJudgement: boolean;
+    /** Reason to show when canRetryJudgement is false. */
+    retryJudgementDisabledReason?: string;
+    /** Number of judge-failed test cases (0 when not applicable/unknown). */
+    judgeFailedCount: number;
+}
+/**
+ * Minimum time a run must have been persisted before a Cancel request with
+ * no in-memory executor token is allowed to take the "zombie" fallback path
+ * (mark cancelled directly — see getRunActionVisibility callers in the
+ * cancel routes). Guards the narrow window right after a run is created:
+ * the doc is persisted (and therefore visible to a concurrent Cancel
+ * request) strictly before its executor registers its cancellation token,
+ * so a Cancel that lands in that gap would otherwise mark a run "cancelled"
+ * moments before its own executor starts making progress on it. A brand-new
+ * run is also the case the fallback is LEAST useful for — "zombie" (no
+ * executor anywhere) is far more plausible once a run has been running for
+ * a while than in its first couple of seconds.
+ */
+export declare const ZOMBIE_CANCEL_MIN_AGE_MS = 5000;
+/**
+ * True once a run is old enough that a tokenless Cancel request can safely
+ * assume its executor (if any) would already have registered a
+ * cancellation token — i.e. it's safe to treat "no token" as "no executor"
+ * rather than "executor hasn't started yet".
+ *
+ * NOTE — known limitation, not fixed by this check: cancellation tokens are
+ * tracked in an in-memory `Map` scoped to ONE server process. In a
+ * multi-process/clustered deployment, a Cancel request routed to a
+ * DIFFERENT process than the one executing the run will always find no
+ * token there, regardless of run age, and this zombie fallback will mark
+ * the run cancelled in storage even though it's alive and progressing on
+ * another process. This mirrors a pre-existing, documented constraint of
+ * this codebase's run-execution model (see AGENTS.md's "orphan-run
+ * recovery" notes: "active is tracked per-process"); fixing it for real
+ * needs the same heartbeat-based ownership (`run.heartbeatAt`) that doc
+ * already calls out as the eventual replacement. Out of scope here.
+ */
+export declare function isOldEnoughForZombieCancel(createdAt: string | undefined, now?: number): boolean;
+/**
+ * Compute the full action-visibility matrix for one run. Pure function —
+ * safe to call from both React components and server-side route validation
+ * so the two never drift.
+ */
+export declare function getRunActionVisibility(run: RunLike | null | undefined): RunActionVisibility;
+//# sourceMappingURL=runActions.d.ts.map

package/lib/dist/lib/runActions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runActions.d.ts","sourceRoot":"","sources":["../../runActions.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAG3D,kFAAkF;AAClF,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,GAAG,aAAa,EAAE,QAAQ,GAAG,SAAS,CAAC,GAAG;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,GAAG,IAAI,aAAa,CAErF;AAED,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAErE;AAED,4EAA4E;AAC5E,wBAAgB,aAAa,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAEtE;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,CAUxE;AAED,MAAM,WAAW,mBAAmB;IAClC,4DAA4D;IAC5D,SAAS,EAAE,OAAO,CAAC;IACnB,kEAAkE;IAClE,SAAS,EAAE,OAAO,CAAC;IACnB,iEAAiE;IACjE,QAAQ,EAAE,OAAO,CAAC;IAClB,+EAA+E;IAC/E,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,4EAA4E;IAC5E,iBAAiB,EAAE,OAAO,CAAC;IAC3B,sDAAsD;IACtD,4BAA4B,CAAC,EAAE,MAAM,CAAC;IACtC,yEAAyE;IACzE,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAOD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAmB,GAAG,OAAO,CAI3G;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,GAAG,SAAS,GAAG,mBAAmB,CAuB3F"}

package/lib/dist/lib/runActions.js

@@ -0,0 +1,130 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { isEvaluationRun as isEvaluationRunDoc } from '../types/index.js';
+/**
+ * True when `run` is a top-level EvaluationRun document (created via
+ * `POST /api/storage/evaluation-runs`), as opposed to a legacy
+ * benchmark-embedded BenchmarkRun (`benchmark.runs[]`). The two share a lot
+ * of shape but only EvaluationRun docs carry `docType: 'evaluation-run'` and
+ * support the rerun/retry-judgement endpoints.
+ *
+ * Null-tolerant wrapper over the typed predicate in `types/index.ts` (the
+ * single source of truth for the docType discriminator) — kept so callers
+ * holding a possibly-null run don't need their own guard.
+ */
+export function isEvaluationRun(run) {
+    return !!run && isEvaluationRunDoc(run);
+}
+/** True while the run has an in-progress executor that a Cancel action could stop. */
+export function isRunRunning(run) {
+    return run?.status === 'running';
+}
+/** True once a run has reached any terminal state (not running/pending). */
+export function isRunTerminal(run) {
+    return !!run && (run.status === 'completed' || run.status === 'failed' || run.status === 'cancelled');
+}
+/**
+ * Count test cases where the AGENT finished (`status === 'completed'`) but
+ * the JUDGE produced no verdict (`passFailStatus` neither 'passed' nor
+ * 'failed') — the "errored" bucket of `lib/runStats` `bucketRunResults`
+ * (issue #242) and exactly the set `POST .../retry-judgement` (default
+ * `scope=errored`) will re-judge. Deliberately excludes:
+ *   - `status !== 'completed'` (agent-failed/cancelled/pending cases — no
+ *     trajectory to re-judge, or nothing ran).
+ *   - a real 'failed' verdict — the judge DID run and graded the case; that
+ *     is a legitimate result, not a judge failure (re-grading it is
+ *     `scope=all`, opt-in from the inspector's dedicated button).
+ *
+ * `passFailStatus` isn't declared on `EvaluationRun['results']`'s static
+ * type (a pre-existing gap — evaluationRunner.ts writes it via an `as any`
+ * spread) so this reads it defensively.
+ */
+export function countJudgeFailed(run) {
+    if (!run?.results)
+        return 0;
+    let count = 0;
+    for (const r of Object.values(run.results)) {
+        const result = r;
+        if (result.status !== 'completed')
+            continue;
+        if (result.passFailStatus === 'passed' || result.passFailStatus === 'failed')
+            continue;
+        count++;
+    }
+    return count;
+}
+const RERUN_NOT_SUPPORTED_REASON = "Re-run isn't available for legacy benchmark-embedded runs";
+const RETRY_JUDGEMENT_NOT_SUPPORTED_REASON = "Retry judgement isn't available for legacy benchmark-embedded runs";
+const RETRY_JUDGEMENT_STILL_RUNNING_REASON = 'Retry judgement is only available once the run finishes';
+const RETRY_JUDGEMENT_NONE_FAILED_REASON = 'No judge-failed test cases to retry';
+/**
+ * Minimum time a run must have been persisted before a Cancel request with
+ * no in-memory executor token is allowed to take the "zombie" fallback path
+ * (mark cancelled directly — see getRunActionVisibility callers in the
+ * cancel routes). Guards the narrow window right after a run is created:
+ * the doc is persisted (and therefore visible to a concurrent Cancel
+ * request) strictly before its executor registers its cancellation token,
+ * so a Cancel that lands in that gap would otherwise mark a run "cancelled"
+ * moments before its own executor starts making progress on it. A brand-new
+ * run is also the case the fallback is LEAST useful for — "zombie" (no
+ * executor anywhere) is far more plausible once a run has been running for
+ * a while than in its first couple of seconds.
+ */
+export const ZOMBIE_CANCEL_MIN_AGE_MS = 5000;
+/**
+ * True once a run is old enough that a tokenless Cancel request can safely
+ * assume its executor (if any) would already have registered a
+ * cancellation token — i.e. it's safe to treat "no token" as "no executor"
+ * rather than "executor hasn't started yet".
+ *
+ * NOTE — known limitation, not fixed by this check: cancellation tokens are
+ * tracked in an in-memory `Map` scoped to ONE server process. In a
+ * multi-process/clustered deployment, a Cancel request routed to a
+ * DIFFERENT process than the one executing the run will always find no
+ * token there, regardless of run age, and this zombie fallback will mark
+ * the run cancelled in storage even though it's alive and progressing on
+ * another process. This mirrors a pre-existing, documented constraint of
+ * this codebase's run-execution model (see AGENTS.md's "orphan-run
+ * recovery" notes: "active is tracked per-process"); fixing it for real
+ * needs the same heartbeat-based ownership (`run.heartbeatAt`) that doc
+ * already calls out as the eventual replacement. Out of scope here.
+ */
+export function isOldEnoughForZombieCancel(createdAt, now = Date.now()) {
+    const created = createdAt ? Date.parse(createdAt) : NaN;
+    if (Number.isNaN(created))
+        return true; // no timestamp to compare against — don't block on it
+    return now - created >= ZOMBIE_CANCEL_MIN_AGE_MS;
+}
+/**
+ * Compute the full action-visibility matrix for one run. Pure function —
+ * safe to call from both React components and server-side route validation
+ * so the two never drift.
+ */
+export function getRunActionVisibility(run) {
+    const evalRun = isEvaluationRun(run);
+    const running = isRunRunning(run);
+    const terminal = isRunTerminal(run);
+    const judgeFailedCount = evalRun ? countJudgeFailed(run) : 0;
+    const canRetryJudgement = evalRun && terminal && judgeFailedCount > 0;
+    let retryJudgementDisabledReason;
+    if (!canRetryJudgement) {
+        if (!evalRun)
+            retryJudgementDisabledReason = RETRY_JUDGEMENT_NOT_SUPPORTED_REASON;
+        else if (!terminal)
+            retryJudgementDisabledReason = RETRY_JUDGEMENT_STILL_RUNNING_REASON;
+        else
+            retryJudgementDisabledReason = RETRY_JUDGEMENT_NONE_FAILED_REASON;
+    }
+    return {
+        canDelete: true,
+        canCancel: running,
+        canRerun: evalRun,
+        rerunDisabledReason: evalRun ? undefined : RERUN_NOT_SUPPORTED_REASON,
+        canRetryJudgement,
+        retryJudgementDisabledReason,
+        judgeFailedCount,
+    };
+}
+//# sourceMappingURL=runActions.js.map

package/lib/dist/lib/runActions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runActions.js","sourceRoot":"","sources":["../../runActions.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8BH,OAAO,EAAE,eAAe,IAAI,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAOhE;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,GAA+B;IAC7D,OAAO,CAAC,CAAC,GAAG,IAAI,kBAAkB,CAAC,GAAmC,CAAC,CAAC;AAC1E,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,GAA+B;IAC1D,OAAO,GAAG,EAAE,MAAM,KAAK,SAAS,CAAC;AACnC,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,aAAa,CAAC,GAA+B;IAC3D,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;AACxG,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAA+B;IAC9D,IAAI,CAAC,GAAG,EAAE,OAAO;QAAE,OAAO,CAAC,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAwD,CAAC;QACxE,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QAC5C,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ;YAAE,SAAS;QACvF,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAmBD,MAAM,0BAA0B,GAAG,2DAA2D,CAAC;AAC/F,MAAM,oCAAoC,GAAG,oEAAoE,CAAC;AAClH,MAAM,oCAAoC,GAAG,yDAAyD,CAAC;AACvG,MAAM,kCAAkC,GAAG,qCAAqC,CAAC;AAEjF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AAE7C;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,0BAA0B,CAAC,SAA6B,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;IAChG,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACxD,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sDAAsD;IAC9F,OAAO,GAAG,GAAG,OAAO,IAAI,wBAAwB,CAAC;AACnD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CAAC,GAA+B;IACpE,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7D,MAAM,iBAAiB,GAAG,OAAO,IAAI,QAAQ,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACtE,IAAI,4BAAgD,CAAC;IACrD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,IAAI,CAAC,OAAO;YAAE,4BAA4B,GAAG,oCAAoC,CAAC;aAC7E,IAAI,CAAC,QAAQ;YAAE,4BAA4B,GAAG,oCAAoC,CAAC;;YACnF,4BAA4B,GAAG,kCAAkC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,OAAO;QAClB,QAAQ,EAAE,OAAO;QACjB,mBAAmB,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,0BAA0B;QACrE,iBAAiB;QACjB,4BAA4B;QAC5B,gBAAgB;KACjB,CAAC;AACJ,CAAC"}

package/lib/dist/lib/runInsights.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Deterministic (no-LLM) aggregation helpers for the run-report "insights"
+ * pane (RunInsightsPane.tsx), shown on the bare
+ * `/benchmarks/:benchmarkId/runs/:runId` route when no test case is
+ * selected — see owner feedback on goyamegh/run-report-redesign: "if no
+ * test case is selected, the right side can show an aggregated view ...
+ * why did the failing tests fail — something that is complete info."
+ *
+ * Everything here is a pure function over already-fetched data (report
+ * summaries + test-case categories). No network calls, no LLM calls — v1
+ * is explicitly deterministic-only per the product ask.
+ */
+export interface CategoryStatusRow {
+    category: string;
+    /** Any ResultStatus value; only 'passed' / 'failed' / 'errored' are counted distinctly, everything else falls into the bar's `total` only (pending/running cases). */
+    status: string;
+}
+export interface CategoryBar {
+    category: string;
+    passed: number;
+    failed: number;
+    errored: number;
+    total: number;
+}
+/**
+ * Group rows by test-case category and tally pass/fail/errored/total.
+ * Deterministic order: largest category first, ties broken alphabetically.
+ */
+export declare function computeCategoryBars(rows: CategoryStatusRow[]): CategoryBar[];
+/**
+ * Normalize a judge-reasoning string down to its first sentence,
+ * lowercased, punctuation-stripped, whitespace-collapsed. Used both as the
+ * clustering input and as the theme's stable `key`.
+ */
+export declare function normalizeReasoningKey(reasoning: string): string;
+export interface FailureThemeInput {
+    testCaseId: string;
+    reasoning: string;
+}
+export interface FailureTheme {
+    /** Stable cluster key — the normalized first sentence of the theme's representative case. */
+    key: string;
+    count: number;
+    /** Trimmed, human-readable first sentence sampled from the theme's most common exact phrasing. */
+    sampleSnippet: string;
+    testCaseIds: string[];
+}
+/**
+ * Cluster failing test cases into "why they failed" themes using a
+ * deterministic, LLM-free heuristic: normalize each case's judge-reasoning
+ * first sentence, then union-find cases that share at least one contiguous
+ * N-word shingle. This is robust to minor paraphrasing (a judge saying
+ * "unable to retrieve" vs "failed to retrieve" the same underlying tool
+ * connectivity failure) while still keeping genuinely distinct failure
+ * modes (e.g. "missing required facts" vs "MCP server unavailable")
+ * separate, because they share no contiguous phrase.
+ *
+ * Verified against a real production run (418-verify, 64 failing cases):
+ * 57 of 64 connectivity-flavored reasonings collapse into ONE dominant
+ * theme; the remaining 7 ("Required facts evaluation: ...", a genuinely
+ * different failure shape) form a second, correctly separate theme.
+ *
+ * Output order: largest theme first, ties broken by the theme's
+ * lowest-sorting testCaseId (deterministic, no dependency on input order).
+ */
+export declare function clusterFailureThemes(items: FailureThemeInput[], shingleSize?: number): FailureTheme[];
+/**
+ * "Based on N of M failing cases" note shown under the theme list when the
+ * reasoning fetch was capped (RunInsightsPane caps at the first 100 failing
+ * cases). Returns null when nothing was capped (fetchedCount >= totalCount).
+ */
+export declare function formatCappedNote(fetchedCount: number, totalFailingCount: number): string | null;
+export interface RankedCase {
+    testCaseId: string;
+    value: number;
+}
+/**
+ * Top-N cases by a numeric value (duration, cost, ...), descending.
+ * Cases with a null/undefined/NaN value are excluded. Deterministic tie
+ * break: lower testCaseId first.
+ */
+export declare function pickTopN(cases: {
+    testCaseId: string;
+    value: number | null | undefined;
+}[], n: number): RankedCase[];
+//# sourceMappingURL=runInsights.d.ts.map

package/lib/dist/lib/runInsights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runInsights.d.ts","sourceRoot":"","sources":["../../runInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;GAWG;AAIH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,sKAAsK;IACtK,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAID;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,WAAW,EAAE,CAiB5E;AAID;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAM/D;AAqBD,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,6FAA6F;IAC7F,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,kGAAkG;IAClG,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAOD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,iBAAiB,EAAE,EAC1B,WAAW,GAAE,MAA6B,GACzC,YAAY,EAAE,CAwEhB;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAG/F;AAID,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAA;CAAE,EAAE,EAAE,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAKnH"}