@opensearch-project/agent-health 0.3.0 → 0.5.2

package/lib/dist/lib/preferences.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Cross-page user preferences.
+ *
+ * Each entry below is a single localStorage key (under the
+ * `agent-health:` prefix) shared by every page that exposes the
+ * corresponding control, so e.g. picking "30d" on the Eval Runs page is
+ * also reflected on Benchmarks and Test Cases.
+ *
+ * Storage layout is intentionally a flat, JSON-serialisable key/value
+ * map: each preference is its own localStorage entry. That keeps the
+ * format portable so prefs can be exported, copied across environments,
+ * or backed up to a server store later without rewriting call sites.
+ *
+ * Helpers `getPreferencesSnapshot()` / `applyPreferencesSnapshot()`
+ * below give a single-call dump and restore for that bundle, which is
+ * the natural shape to send over the network.
+ */
+/** Stable, user-facing namespace for all cross-page preferences. */
+export declare const PREFS_KEYS: {
+    /** Run-config: agent the user wants to RUN (QuickRunModal, NewRunPage). */
+    readonly agentKey: "prefs:agentKey";
+    /** Run-config: model the user wants to RUN (QuickRunModal, NewRunPage). */
+    readonly modelId: "prefs:modelId";
+    /** Time-range filter, shared by evaluation list pages and Agent Traces. */
+    readonly timeRange: "prefs:timeRange";
+    /** Agent filter dropdown shared by Benchmarks and Eval Runs. Defaults to `'all'`. */
+    readonly agentFilter: "prefs:agentFilter";
+    /** Benchmark filter dropdown shared by Test Cases (and any future page). */
+    readonly benchmarkFilter: "prefs:benchmarkFilter";
+    /** Flat / grouped view mode shared by list pages. */
+    readonly viewMode: "prefs:viewMode";
+};
+/** Possible string values for {@link PREFS_KEYS.timeRange}. */
+export type SharedTimeRange = '1h' | '6h' | '1d' | '7d' | '30d' | 'all';
+/**
+ * Convert a {@link SharedTimeRange} to a "minutes ago" cutoff suitable
+ * for the AgentTraces query. `'all'` is mapped to a very large window
+ * (90d) so that there's always *some* finite cutoff sent to the server.
+ */
+export declare function sharedTimeRangeToMinutes(range: SharedTimeRange): number;
+/**
+ * Return a portable snapshot of every cross-page preference currently
+ * stored in localStorage. Keys are the unprefixed identifiers (e.g.
+ * `'prefs:timeRange'`) and values are already JSON-decoded.
+ *
+ * Use this to copy a user's preferences to clipboard, persist them
+ * server-side, or assert against them in tests.
+ */
+export declare function getPreferencesSnapshot(): Record<string, unknown>;
+/**
+ * Apply a snapshot produced by {@link getPreferencesSnapshot}.
+ * Unknown keys are ignored so the snapshot format stays forward-
+ * compatible with future preferences. Pass an empty object to clear
+ * every preference (sets nothing — call {@link clearPreferences} for
+ * an explicit reset).
+ */
+export declare function applyPreferencesSnapshot(snapshot: Record<string, unknown>): void;
+/**
+ * Remove every preference from localStorage. Useful for "Reset
+ * preferences" buttons and tests.
+ */
+export declare function clearPreferences(): void;
+//# sourceMappingURL=preferences.d.ts.map

package/lib/dist/lib/preferences.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preferences.d.ts","sourceRoot":"","sources":["../../preferences.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;;;;GAgBG;AAEH,oEAAoE;AACpE,eAAO,MAAM,UAAU;IACrB,2EAA2E;;IAE3E,2EAA2E;;IAE3E,2EAA2E;;IAE3E,qFAAqF;;IAErF,4EAA4E;;IAE5E,qDAAqD;;CAE7C,CAAC;AAEX,+DAA+D;AAC/D,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,CAAC;AAExE;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CASvE;AAID;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAchE;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAWhF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CASvC"}

package/lib/dist/lib/preferences.js

@@ -0,0 +1,117 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Cross-page user preferences.
+ *
+ * Each entry below is a single localStorage key (under the
+ * `agent-health:` prefix) shared by every page that exposes the
+ * corresponding control, so e.g. picking "30d" on the Eval Runs page is
+ * also reflected on Benchmarks and Test Cases.
+ *
+ * Storage layout is intentionally a flat, JSON-serialisable key/value
+ * map: each preference is its own localStorage entry. That keeps the
+ * format portable so prefs can be exported, copied across environments,
+ * or backed up to a server store later without rewriting call sites.
+ *
+ * Helpers `getPreferencesSnapshot()` / `applyPreferencesSnapshot()`
+ * below give a single-call dump and restore for that bundle, which is
+ * the natural shape to send over the network.
+ */
+/** Stable, user-facing namespace for all cross-page preferences. */
+export const PREFS_KEYS = {
+    /** Run-config: agent the user wants to RUN (QuickRunModal, NewRunPage). */
+    agentKey: 'prefs:agentKey',
+    /** Run-config: model the user wants to RUN (QuickRunModal, NewRunPage). */
+    modelId: 'prefs:modelId',
+    /** Time-range filter, shared by evaluation list pages and Agent Traces. */
+    timeRange: 'prefs:timeRange',
+    /** Agent filter dropdown shared by Benchmarks and Eval Runs. Defaults to `'all'`. */
+    agentFilter: 'prefs:agentFilter',
+    /** Benchmark filter dropdown shared by Test Cases (and any future page). */
+    benchmarkFilter: 'prefs:benchmarkFilter',
+    /** Flat / grouped view mode shared by list pages. */
+    viewMode: 'prefs:viewMode',
+};
+/**
+ * Convert a {@link SharedTimeRange} to a "minutes ago" cutoff suitable
+ * for the AgentTraces query. `'all'` is mapped to a very large window
+ * (90d) so that there's always *some* finite cutoff sent to the server.
+ */
+export function sharedTimeRangeToMinutes(range) {
+    switch (range) {
+        case '1h': return 60;
+        case '6h': return 360;
+        case '1d': return 1440;
+        case '7d': return 10080;
+        case '30d': return 43200;
+        case 'all': return 60 * 24 * 90; // 90 days as a practical "all"
+    }
+}
+const STORAGE_PREFIX = 'agent-health:';
+/**
+ * Return a portable snapshot of every cross-page preference currently
+ * stored in localStorage. Keys are the unprefixed identifiers (e.g.
+ * `'prefs:timeRange'`) and values are already JSON-decoded.
+ *
+ * Use this to copy a user's preferences to clipboard, persist them
+ * server-side, or assert against them in tests.
+ */
+export function getPreferencesSnapshot() {
+    if (typeof window === 'undefined')
+        return {};
+    const out = {};
+    for (const key of Object.values(PREFS_KEYS)) {
+        try {
+            const raw = localStorage.getItem(STORAGE_PREFIX + key);
+            if (raw === null)
+                continue;
+            out[key] = JSON.parse(raw);
+        }
+        catch {
+            // Skip entries we can't decode; the caller will treat them as
+            // missing and the corresponding page will fall back to its default.
+        }
+    }
+    return out;
+}
+/**
+ * Apply a snapshot produced by {@link getPreferencesSnapshot}.
+ * Unknown keys are ignored so the snapshot format stays forward-
+ * compatible with future preferences. Pass an empty object to clear
+ * every preference (sets nothing — call {@link clearPreferences} for
+ * an explicit reset).
+ */
+export function applyPreferencesSnapshot(snapshot) {
+    if (typeof window === 'undefined')
+        return;
+    const known = new Set(Object.values(PREFS_KEYS));
+    for (const [key, value] of Object.entries(snapshot)) {
+        if (!known.has(key))
+            continue;
+        try {
+            localStorage.setItem(STORAGE_PREFIX + key, JSON.stringify(value));
+        }
+        catch {
+            // Quota / storage disabled — ignore.
+        }
+    }
+}
+/**
+ * Remove every preference from localStorage. Useful for "Reset
+ * preferences" buttons and tests.
+ */
+export function clearPreferences() {
+    if (typeof window === 'undefined')
+        return;
+    for (const key of Object.values(PREFS_KEYS)) {
+        try {
+            localStorage.removeItem(STORAGE_PREFIX + key);
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+//# sourceMappingURL=preferences.js.map

package/lib/dist/lib/preferences.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"preferences.js","sourceRoot":"","sources":["../../preferences.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;GAgBG;AAEH,oEAAoE;AACpE,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,2EAA2E;IAC3E,QAAQ,EAAE,gBAAgB;IAC1B,2EAA2E;IAC3E,OAAO,EAAE,eAAe;IACxB,2EAA2E;IAC3E,SAAS,EAAE,iBAAiB;IAC5B,qFAAqF;IACrF,WAAW,EAAE,mBAAmB;IAChC,4EAA4E;IAC5E,eAAe,EAAE,uBAAuB;IACxC,qDAAqD;IACrD,QAAQ,EAAE,gBAAgB;CAClB,CAAC;AAKX;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAsB;IAC7D,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC;QACtB,KAAK,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC;QACvB,KAAK,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC;QACxB,KAAK,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC;QACzB,KAAK,KAAK,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,+BAA+B;IAClE,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,eAAe,CAAC;AAEvC;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB;IACpC,IAAI,OAAO,MAAM,KAAK,WAAW;QAAE,OAAO,EAAE,CAAC;IAC7C,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;YACvD,IAAI,GAAG,KAAK,IAAI;gBAAE,SAAS;YAC3B,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,oEAAoE;QACtE,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,QAAiC;IACxE,IAAI,OAAO,MAAM,KAAK,WAAW;QAAE,OAAO;IAC1C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;IACzD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC9B,IAAI,CAAC;YACH,YAAY,CAAC,OAAO,CAAC,cAAc,GAAG,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB;IAC9B,IAAI,OAAO,MAAM,KAAK,WAAW;QAAE,OAAO;IAC1C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,YAAY,CAAC,UAAU,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;AACH,CAAC"}

package/lib/dist/lib/resolveAgentModel.d.ts

@@ -0,0 +1,22 @@
+import type { AgentConfig } from '../types/index.js';
+/**
+ * Resolve the LLM an agent runs on.
+ *
+ * The agent's model is owned by the agent's OWN config (`agent-health.config.ts`),
+ * NOT a user-facing "agent model" selector. There is intentionally no run-level
+ * or UI/CLI way to override it — picking a model separately from the agent is a
+ * leaky abstraction (subprocess agents like Claude Code / pi ignore it and run
+ * on whatever their connector is configured with). Claude-like agents that DO
+ * accept a model take it via their connector config.
+ *
+ * Resolution order (all from the agent's `connectorConfig`):
+ *   1. `connectorConfig.model`              — pi / streaming / openai-compatible
+ *   2. `connectorConfig.env.ANTHROPIC_MODEL` — claude-code
+ *   3. `connectorConfig.args` `--model <id>` — any subprocess agent passing a flag
+ *
+ * `fallback` exists only for backward compatibility with runs/agents created
+ * before the agent-model concept was removed (where the run carried a
+ * user-selected `modelId`). New code should not rely on it.
+ */
+export declare function resolveAgentModel(agent: AgentConfig | undefined | null, fallback?: string): string;
+//# sourceMappingURL=resolveAgentModel.d.ts.map

package/lib/dist/lib/resolveAgentModel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveAgentModel.d.ts","sourceRoot":"","sources":["../../resolveAgentModel.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,EACrC,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,CAWR"}

package/lib/dist/lib/resolveAgentModel.js

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright OpenSearch Contributors
+/**
+ * Resolve the LLM an agent runs on.
+ *
+ * The agent's model is owned by the agent's OWN config (`agent-health.config.ts`),
+ * NOT a user-facing "agent model" selector. There is intentionally no run-level
+ * or UI/CLI way to override it — picking a model separately from the agent is a
+ * leaky abstraction (subprocess agents like Claude Code / pi ignore it and run
+ * on whatever their connector is configured with). Claude-like agents that DO
+ * accept a model take it via their connector config.
+ *
+ * Resolution order (all from the agent's `connectorConfig`):
+ *   1. `connectorConfig.model`              — pi / streaming / openai-compatible
+ *   2. `connectorConfig.env.ANTHROPIC_MODEL` — claude-code
+ *   3. `connectorConfig.args` `--model <id>` — any subprocess agent passing a flag
+ *
+ * `fallback` exists only for backward compatibility with runs/agents created
+ * before the agent-model concept was removed (where the run carried a
+ * user-selected `modelId`). New code should not rely on it.
+ */
+export function resolveAgentModel(agent, fallback) {
+    const cc = (agent?.connectorConfig || {});
+    if (typeof cc.model === 'string' && cc.model)
+        return cc.model;
+    const envModel = cc.env?.ANTHROPIC_MODEL;
+    if (typeof envModel === 'string' && envModel)
+        return envModel;
+    const args = cc.args;
+    if (Array.isArray(args)) {
+        const i = args.indexOf('--model');
+        if (i >= 0 && typeof args[i + 1] === 'string' && args[i + 1])
+            return args[i + 1];
+    }
+    return fallback || '';
+}
+//# sourceMappingURL=resolveAgentModel.js.map

package/lib/dist/lib/resolveAgentModel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveAgentModel.js","sourceRoot":"","sources":["../../resolveAgentModel.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,oCAAoC;AAIpC;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAqC,EACrC,QAAiB;IAEjB,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,eAAe,IAAI,EAAE,CAAwB,CAAC;IACjE,IAAI,OAAO,EAAE,CAAC,KAAK,KAAK,QAAQ,IAAI,EAAE,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC,KAAK,CAAC;IAC9D,MAAM,QAAQ,GAAG,EAAE,CAAC,GAAG,EAAE,eAAe,CAAC;IACzC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC9D,MAAM,IAAI,GAAY,EAAE,CAAC,IAAI,CAAC;IAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,CAAW,CAAC;IAC7F,CAAC;IACD,OAAO,QAAQ,IAAI,EAAE,CAAC;AACxB,CAAC"}

package/lib/dist/lib/runStats.d.ts

@@ -0,0 +1,92 @@
+/**
+ * Shared Run Statistics Calculation
+ *
+ * This module provides shared logic for calculating benchmark run statistics.
+ * Both UI and CLI use these functions to ensure consistent pass/fail counting.
+ *
+ * The canonical approach is:
+ * 1. Get report IDs from run.results[testCaseId].reportId
+ * 2. Fetch each report by ID
+ * 3. Count passFailStatus from the fetched reports
+ *
+ * For performance optimization, stats are denormalized onto BenchmarkRun.stats
+ * and updated incrementally as test cases complete.
+ */
+import type { BenchmarkRun, EvaluationReport, RunStats as RunStatsType } from '../types/index.js';
+/**
+ * Statistics for a benchmark run
+ */
+export interface RunStats {
+    /** Number of test cases that passed (passFailStatus === 'passed') */
+    passed: number;
+    /** Number of test cases that failed (passFailStatus === 'failed' or execution failed) */
+    failed: number;
+    /** Number of test cases still pending (running, or report not yet available) */
+    pending: number;
+    /**
+     * Number of test cases where the *evaluator* could not produce a verdict
+     * (judge validation error, trace timeout, etc.). Excluded from passed and
+     * failed counts so a misconfigured evaluator doesn't poison aggregate
+     * pass rates. Issue #242.
+     */
+    errored: number;
+    /** Total number of test cases in the run */
+    total: number;
+    /**
+     * Pass rate as a percentage (0-100). Computed over `total - errored`
+     * (the *evaluable* set), not over `total`, so a non-retryable judge
+     * failure can't masquerade as the agent scoring 0%.
+     */
+    passRate: number;
+}
+/**
+ * Bucket a run's per-test-case results into passed/failed/errored/pending using
+ * ONLY the persisted result fields (status + passFailStatus) — no reports
+ * needed. This is the SINGLE source of truth for pass/fail/errored counts across
+ * the app (the runs list AND the comparison page), so the numbers can't
+ * diverge between views.
+ *
+ * The denormalized `run.stats` is NOT authoritative: its writer historically
+ * counted every 'completed' result as passed without checking the verdict, so
+ * an errored case (judge produced no verdict) was miscounted as a pass and
+ * `errored` was never tracked. Recompute from results instead.
+ *
+ * Errored (#242): a 'completed' result with no 'passed'/'failed' verdict means
+ * the evaluator couldn't produce one (judge validation error, trace timeout).
+ * Excluded from passed/failed — exactly as calculateRunStats does via the
+ * report's metricsStatus.
+ */
+export declare function bucketRunResults(results: Record<string, {
+    status?: string;
+    passFailStatus?: string;
+}> | undefined): Pick<RunStats, 'passed' | 'failed' | 'errored' | 'pending' | 'total'>;
+/**
+ * Calculate statistics for a benchmark run.
+ *
+ * This function uses the same logic as the UI to count pass/fail status:
+ * - Iterates over run.results to get reportIds
+ * - Looks up each report in the provided reports map
+ * - Counts passFailStatus from completed reports
+ *
+ * @param run - The benchmark run to calculate stats for
+ * @param reports - Map of reportId -> EvaluationReport (pre-fetched)
+ * @returns Calculated statistics including passed, failed, pending, total, and passRate
+ */
+export declare function calculateRunStats(run: BenchmarkRun, reports: Record<string, EvaluationReport | null>): RunStats;
+/**
+ * Extract all report IDs from a run's results that need to be fetched.
+ *
+ * @param run - The benchmark run to extract report IDs from
+ * @returns Array of unique report IDs
+ */
+export declare function getReportIdsFromRun(run: BenchmarkRun): string[];
+/**
+ * Compute stats from a benchmark run and its reports.
+ * This is the server-side version that returns the denormalized stats object.
+ *
+ * @param run - The benchmark run to compute stats for
+ * @param reports - Array of reports for this run
+ * @returns Denormalized stats object (passed, failed, pending, total)
+ */
+export declare function computeRunStatsFromReports(run: BenchmarkRun, reports: EvaluationReport[]): RunStatsType;
+//# sourceMappingURL=runStats.d.ts.map

package/lib/dist/lib/runStats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runStats.d.ts","sourceRoot":"","sources":["../../runStats.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,IAAI,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAEjG;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,yFAAyF;IACzF,MAAM,EAAE,MAAM,CAAC;IACf,gFAAgF;IAChF,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,SAAS,GAChF,IAAI,CAAC,QAAQ,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC,CAevE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,YAAY,EACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAC,GAC/C,QAAQ,CAuEV;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,EAAE,CAU/D;AAED;;;;;;;GAOG;AACH,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,YAAY,EACjB,OAAO,EAAE,gBAAgB,EAAE,GAC1B,YAAY,CAed"}

package/lib/dist/lib/runStats.js

@@ -0,0 +1,160 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Bucket a run's per-test-case results into passed/failed/errored/pending using
+ * ONLY the persisted result fields (status + passFailStatus) — no reports
+ * needed. This is the SINGLE source of truth for pass/fail/errored counts across
+ * the app (the runs list AND the comparison page), so the numbers can't
+ * diverge between views.
+ *
+ * The denormalized `run.stats` is NOT authoritative: its writer historically
+ * counted every 'completed' result as passed without checking the verdict, so
+ * an errored case (judge produced no verdict) was miscounted as a pass and
+ * `errored` was never tracked. Recompute from results instead.
+ *
+ * Errored (#242): a 'completed' result with no 'passed'/'failed' verdict means
+ * the evaluator couldn't produce one (judge validation error, trace timeout).
+ * Excluded from passed/failed — exactly as calculateRunStats does via the
+ * report's metricsStatus.
+ */
+export function bucketRunResults(results) {
+    let passed = 0, failed = 0, errored = 0, pending = 0, total = 0;
+    for (const r of Object.values(results || {})) {
+        total++;
+        if (r.status === 'pending' || r.status === 'running') {
+            pending++;
+            continue;
+        }
+        if (r.status === 'failed' || r.status === 'cancelled') {
+            failed++;
+            continue;
+        }
+        if (r.status === 'completed') {
+            if (r.passFailStatus === 'passed')
+                passed++;
+            else if (r.passFailStatus === 'failed')
+                failed++;
+            else
+                errored++; // completed without a verdict = judge errored (#242)
+            continue;
+        }
+        pending++; // unknown / no status
+    }
+    return { passed, failed, errored, pending, total };
+}
+/**
+ * Calculate statistics for a benchmark run.
+ *
+ * This function uses the same logic as the UI to count pass/fail status:
+ * - Iterates over run.results to get reportIds
+ * - Looks up each report in the provided reports map
+ * - Counts passFailStatus from completed reports
+ *
+ * @param run - The benchmark run to calculate stats for
+ * @param reports - Map of reportId -> EvaluationReport (pre-fetched)
+ * @returns Calculated statistics including passed, failed, pending, total, and passRate
+ */
+export function calculateRunStats(run, reports) {
+    let passed = 0;
+    let failed = 0;
+    let pending = 0;
+    let errored = 0;
+    let total = 0;
+    Object.entries(run.results || {}).forEach(([testCaseId, result]) => {
+        total++;
+        // Check result status first
+        if (result.status === 'pending' || result.status === 'running') {
+            pending++;
+            return;
+        }
+        if (result.status === 'failed' || result.status === 'cancelled') {
+            failed++;
+            return;
+        }
+        // For completed results, check the report's passFailStatus
+        if (result.status === 'completed' && result.reportId) {
+            const report = reports[result.reportId];
+            if (!report) {
+                // Report not loaded yet or doesn't exist
+                pending++;
+                return;
+            }
+            // Check if evaluation is still pending (trace mode)
+            if (report.metricsStatus === 'pending' || report.metricsStatus === 'calculating') {
+                pending++;
+                return;
+            }
+            // Issue #242: evaluator could not produce a verdict (judge validation
+            // error, trace timeout, etc.). Excluded from passed/failed so
+            // misconfigured evaluators don't masquerade as agent failures.
+            if (report.metricsStatus === 'error') {
+                errored++;
+                return;
+            }
+            // Count based on passFailStatus from LLM judge
+            if (report.passFailStatus === 'passed') {
+                passed++;
+            }
+            else {
+                // passFailStatus === 'failed' or undefined (treat as failed)
+                failed++;
+            }
+        }
+        else {
+            // No reportId but status is completed - treat as pending
+            pending++;
+        }
+    });
+    // Pass rate is computed over the evaluable set (total minus errored).
+    // If every run errored, expose 0% rather than dividing by zero.
+    const evaluable = Math.max(0, total - errored);
+    const passRate = evaluable > 0 ? Math.round((passed / evaluable) * 100) : 0;
+    return {
+        passed,
+        failed,
+        pending,
+        errored,
+        total,
+        passRate,
+    };
+}
+/**
+ * Extract all report IDs from a run's results that need to be fetched.
+ *
+ * @param run - The benchmark run to extract report IDs from
+ * @returns Array of unique report IDs
+ */
+export function getReportIdsFromRun(run) {
+    const reportIds = new Set();
+    Object.values(run.results || {}).forEach((result) => {
+        if (result.reportId) {
+            reportIds.add(result.reportId);
+        }
+    });
+    return Array.from(reportIds);
+}
+/**
+ * Compute stats from a benchmark run and its reports.
+ * This is the server-side version that returns the denormalized stats object.
+ *
+ * @param run - The benchmark run to compute stats for
+ * @param reports - Array of reports for this run
+ * @returns Denormalized stats object (passed, failed, pending, total)
+ */
+export function computeRunStatsFromReports(run, reports) {
+    const reportsMap = {};
+    reports.forEach(report => {
+        reportsMap[report.id] = report;
+    });
+    const fullStats = calculateRunStats(run, reportsMap);
+    return {
+        passed: fullStats.passed,
+        failed: fullStats.failed,
+        pending: fullStats.pending,
+        errored: fullStats.errored,
+        total: fullStats.total,
+    };
+}
+//# sourceMappingURL=runStats.js.map

package/lib/dist/lib/runStats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runStats.js","sourceRoot":"","sources":["../../runStats.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8CH;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAiF;IAEjF,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;IAChE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;QAC7C,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9E,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAAC,MAAM,EAAE,CAAC;YAAC,SAAS;QAAC,CAAC;QAC9E,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,CAAC,cAAc,KAAK,QAAQ;gBAAE,MAAM,EAAE,CAAC;iBACvC,IAAI,CAAC,CAAC,cAAc,KAAK,QAAQ;gBAAE,MAAM,EAAE,CAAC;;gBAC5C,OAAO,EAAE,CAAC,CAAC,qDAAqD;YACrE,SAAS;QACX,CAAC;QACD,OAAO,EAAE,CAAC,CAAC,sBAAsB;IACnC,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,iBAAiB,CAC/B,GAAiB,EACjB,OAAgD;IAEhD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,EAAE,EAAE;QACjE,KAAK,EAAE,CAAC;QAER,4BAA4B;QAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC/D,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAChE,MAAM,EAAE,CAAC;YACT,OAAO;QACT,CAAC;QAED,2DAA2D;QAC3D,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAExC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,yCAAyC;gBACzC,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YAED,oDAAoD;YACpD,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,aAAa,KAAK,aAAa,EAAE,CAAC;gBACjF,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YAED,sEAAsE;YACtE,8DAA8D;YAC9D,+DAA+D;YAC/D,IAAI,MAAM,CAAC,aAAa,KAAK,OAAO,EAAE,CAAC;gBACrC,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,IAAI,MAAM,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,EAAE,CAAC;YACX,CAAC;iBAAM,CAAC;gBACN,6DAA6D;gBAC7D,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;aAAM,CAAC;YACN,yDAAyD;YACzD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,sEAAsE;IACtE,gEAAgE;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,OAAO;QACL,MAAM;QACN,MAAM;QACN,OAAO;QACP,OAAO;QACP,KAAK;QACL,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAiB;IACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QAClD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAC/B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CACxC,GAAiB,EACjB,OAA2B;IAE3B,MAAM,UAAU,GAA4C,EAAE,CAAC;IAC/D,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;QACvB,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAErD,OAAO;QACL,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,KAAK,EAAE,SAAS,CAAC,KAAK;KACvB,CAAC;AACJ,CAAC"}

package/lib/dist/lib/telemetry/constants.d.ts

@@ -0,0 +1,60 @@
+/**
+ * OTel Semantic Convention constants for evaluation telemetry.
+ *
+ * In-spec attributes are re-exported from @opentelemetry/semantic-conventions.
+ * Proposed and extension attributes are defined here as string constants.
+ */
+export { ATTR_TEST_SUITE_NAME, ATTR_TEST_SUITE_RUN_STATUS, ATTR_TEST_CASE_NAME, ATTR_TEST_CASE_RESULT_STATUS, TEST_SUITE_RUN_STATUS_VALUE_SUCCESS, TEST_SUITE_RUN_STATUS_VALUE_FAILURE, TEST_SUITE_RUN_STATUS_VALUE_IN_PROGRESS, TEST_CASE_RESULT_STATUS_VALUE_PASS, TEST_CASE_RESULT_STATUS_VALUE_FAIL, ATTR_GEN_AI_OPERATION_NAME, ATTR_GEN_AI_EVALUATION_NAME, ATTR_GEN_AI_EVALUATION_SCORE_VALUE, ATTR_GEN_AI_EVALUATION_SCORE_LABEL, ATTR_GEN_AI_EVALUATION_EXPLANATION, EVENT_GEN_AI_EVALUATION_RESULT, } from '@opentelemetry/semantic-conventions/incubating';
+/** Unique identifier for a specific test suite run */
+export declare const ATTR_TEST_SUITE_RUN_ID: "test.suite.run.id";
+/** Machine-readable unique identifier for a test case */
+export declare const ATTR_TEST_CASE_ID: "test.case.id";
+/** Test case input (prompt) */
+export declare const ATTR_TEST_CASE_INPUT: "test.case.input";
+/** Test case output (agent response) */
+export declare const ATTR_TEST_CASE_OUTPUT: "test.case.output";
+/** Test case expected outcome */
+export declare const ATTR_TEST_CASE_EXPECTED: "test.case.expected";
+/** Model ID used by the LLM judge */
+export declare const ATTR_AGENT_HEALTH_JUDGE_MODEL_ID: "agent_health.judge.model_id";
+/** Time spent in LLM judge evaluation (ms) */
+export declare const ATTR_AGENT_HEALTH_JUDGE_DURATION_MS: "agent_health.judge.duration_ms";
+/** Number of retry attempts for the LLM judge */
+export declare const ATTR_AGENT_HEALTH_JUDGE_ATTEMPTS: "agent_health.judge.attempts";
+/** Time spent in agent execution (ms) */
+export declare const ATTR_AGENT_HEALTH_AGENT_DURATION_MS: "agent_health.agent.duration_ms";
+/** Connector protocol used for agent communication */
+export declare const ATTR_AGENT_HEALTH_CONNECTOR_PROTOCOL: "agent_health.connector.protocol";
+/**
+ * Agent run ID — links eval spans to the agent execution trace (Strategy B).
+ *
+ * Uses Agent Health's own namespace rather than the OpenTelemetry-reserved
+ * `gen_ai.*` namespace: `gen_ai.request.id` is not a registered semantic
+ * convention attribute (the `gen_ai.request.*` namespace is for request
+ * parameters), and the OTEL naming spec advises against adding app-specific
+ * keys under an existing OTEL namespace. W3C trace context (shared traceId,
+ * Strategy A) remains the primary correlation; this attribute is the loose
+ * link for agents that don't propagate context.
+ */
+export declare const ATTR_AGENT_HEALTH_AGENT_RUN_ID: "agent_health.run.id";
+/** Operation name value for evaluation spans */
+export declare const GEN_AI_OPERATION_NAME_VALUE_EVALUATION: "evaluation";
+/**
+ * OTEL-registered (incubating) conversation/session identifier —
+ * `gen_ai.conversation.id` (model/gen-ai/registry.yaml): "the unique
+ * identifier for a conversation (session, thread), used to store and correlate
+ * messages within this conversation." We stamp this (= the agent run id) on our
+ * eval + sample-agent spans so trace queries can correlate on the OTEL-standard
+ * attribute in addition to our own `agent_health.run.id`. Declared as a string
+ * literal (rather than re-exported from the incubating module) so it's stable
+ * regardless of semantic-conventions packaging. See AGENTS.md → Trace
+ * correlation conventions.
+ */
+export declare const ATTR_GEN_AI_CONVERSATION_ID: "gen_ai.conversation.id";
+/** Maximum length for truncated attribute values */
+export declare const MAX_ATTRIBUTE_LENGTH = 10000;
+/** Maximum length for judge explanation in events */
+export declare const MAX_EXPLANATION_LENGTH = 4000;
+/** Tracer name used for evaluation spans */
+export declare const EVAL_TRACER_NAME = "agent-health-eval";
+//# sourceMappingURL=constants.d.ts.map

package/lib/dist/lib/telemetry/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../../telemetry/constants.ts"],"names":[],"mappings":"AAKA;;;;;GAKG;AAOH,OAAO,EAEL,oBAAoB,EACpB,0BAA0B,EAC1B,mBAAmB,EACnB,4BAA4B,EAC5B,mCAAmC,EACnC,mCAAmC,EACnC,uCAAuC,EACvC,kCAAkC,EAClC,kCAAkC,EAGlC,0BAA0B,EAC1B,2BAA2B,EAC3B,kCAAkC,EAClC,kCAAkC,EAClC,kCAAkC,EAGlC,8BAA8B,GAC/B,MAAM,gDAAgD,CAAC;AAOxD,sDAAsD;AACtD,eAAO,MAAM,sBAAsB,EAAG,mBAA4B,CAAC;AAEnE,yDAAyD;AACzD,eAAO,MAAM,iBAAiB,EAAG,cAAuB,CAAC;AAMzD,+BAA+B;AAC/B,eAAO,MAAM,oBAAoB,EAAG,iBAA0B,CAAC;AAE/D,wCAAwC;AACxC,eAAO,MAAM,qBAAqB,EAAG,kBAA2B,CAAC;AAEjE,iCAAiC;AACjC,eAAO,MAAM,uBAAuB,EAAG,oBAA6B,CAAC;AAMrE,qCAAqC;AACrC,eAAO,MAAM,gCAAgC,EAAG,6BAAsC,CAAC;AAEvF,8CAA8C;AAC9C,eAAO,MAAM,mCAAmC,EAAG,gCAAyC,CAAC;AAE7F,iDAAiD;AACjD,eAAO,MAAM,gCAAgC,EAAG,6BAAsC,CAAC;AAEvF,yCAAyC;AACzC,eAAO,MAAM,mCAAmC,EAAG,gCAAyC,CAAC;AAE7F,sDAAsD;AACtD,eAAO,MAAM,oCAAoC,EAAG,iCAA0C,CAAC;AAE/F;;;;;;;;;;GAUG;AACH,eAAO,MAAM,8BAA8B,EAAG,qBAA8B,CAAC;AAM7E,gDAAgD;AAChD,eAAO,MAAM,sCAAsC,EAAG,YAAqB,CAAC;AAE5E;;;;;;;;;;GAUG;AACH,eAAO,MAAM,2BAA2B,EAAG,wBAAiC,CAAC;AAE7E,oDAAoD;AACpD,eAAO,MAAM,oBAAoB,QAAQ,CAAC;AAE1C,qDAAqD;AACrD,eAAO,MAAM,sBAAsB,OAAO,CAAC;AAE3C,4CAA4C;AAC5C,eAAO,MAAM,gBAAgB,sBAAsB,CAAC"}