@opensearch-project/agent-health 0.4.0 → 0.6.0

package/lib/dist/lib/comparisonInsights.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Comparison insights — deterministic agreement + category math for the
+ * comparison page's insights band (and, later, as grounding context for the
+ * "What's actually different" LLM deep-dive).
+ *
+ * Everything here is pure arithmetic over the already-loaded comparison rows:
+ * no API calls, no LLM. The band renders the *what* (which runs agree where,
+ * which categories are weak); the deep-dive explains the *why*.
+ *
+ * Generalized to N runs from the start:
+ *   - 2 runs  → "Both pass / Both fail / Split"
+ *   - 3+ runs → "All pass / All fail / Split"
+ */
+import type { TestCaseComparisonRow } from '../types/index.js';
+/** Agreement bucket for a row across the selected runs. */
+export type AgreementBucket = 'allPass' | 'allFail' | 'split';
+export interface AgreementPartition {
+    allPass: TestCaseComparisonRow[];
+    allFail: TestCaseComparisonRow[];
+    split: TestCaseComparisonRow[];
+    /** Rows where at least one run has no verdict (missing report / not run). */
+    uncovered: TestCaseComparisonRow[];
+}
+/**
+ * Partition rows into agreement buckets across the given runs.
+ * A row only participates when EVERY selected run has a verdict for it —
+ * partially-covered rows go to `uncovered` (they can't agree or disagree).
+ */
+export declare function partitionByAgreement(rows: TestCaseComparisonRow[], runIds: string[]): AgreementPartition;
+/** Bucket a single row (same semantics as {@link partitionByAgreement}); null = uncovered. */
+export declare function bucketRow(row: TestCaseComparisonRow, runIds: string[]): AgreementBucket | null;
+/**
+ * Synthetic bucket names. Parentheses are deliberately outside the
+ * name-tag charset (`[\w-]`), so a real benchmark category can never
+ * collide with these rollup buckets.
+ */
+export declare const UNCATEGORIZED = "(uncategorized)";
+/** Rollup bucket for categories with too few cases to be meaningful. */
+export declare const OTHER_CATEGORY = "(other)";
+/**
+ * Extract a row's category.
+ * Priority:
+ *   1. The `subcategory:<x>` label — the proper, purpose-built tag for this
+ *      (set via the SDK's `test(name, { labels: ['subcategory:basic'] })`,
+ *      the JSON/CLI import's `subcategory` field, or the Test Case editor).
+ *      Prefer this over anything scraped from free text: it's validated,
+ *      exported/round-tripped, and shown elsewhere in the UI (EvalsPage,
+ *      BenchmarkEditor) — a real column, not a regex guess.
+ *   2. A bracketed tag embedded in the test-case name — a convention older
+ *      imported benchmarks used before `subcategory` existed (e.g.
+ *      "qst_0011 [basic] How long is …"). Kept only for benchmarks that
+ *      predate the `subcategory` field; new imports should set it instead.
+ *   3. A `topic:<x>` label (the generic labels system, pre-dates
+ *      `subcategory` as a dedicated concept).
+ *   4. {@link UNCATEGORIZED}.
+ * (`category:<x>` labels are intentionally NOT used here: imported benchmarks
+ * stamp a single `category:RAG` on every case, which would collapse the
+ * breakdown into one column — that's the coarse domain, not this facet.)
+ */
+export declare function extractRowCategory(row: Pick<TestCaseComparisonRow, 'testCaseName' | 'labels'>): string;
+export interface CategoryCell {
+    passed: number;
+    total: number;
+}
+export interface CategoryBreakdown {
+    /** Display order: by case count desc; `other` (if present) always last. */
+    categories: string[];
+    /** runId → category → {passed,total}. Totals may differ per run (missing results are skipped). */
+    perRun: Record<string, Record<string, CategoryCell>>;
+    /** Overall case count per category (across rows, not per run). */
+    totals: Record<string, number>;
+    /**
+     * Raw categories each displayed column represents. Identity for real
+     * categories; the union of rolled-up raw categories for `(other)`.
+     * Filtering MUST use this mapping so clicking `(other)` matches exactly
+     * the rows its cell counted.
+     */
+    members: Record<string, string[]>;
+}
+/**
+ * Default minimum case count for a category to get its own column; smaller
+ * ones roll up into `other`. Callers with small datasets may lower this —
+ * a cell needs enough cases that a single verdict flip doesn't swing it
+ * by tens of percentage points.
+ */
+export declare const MIN_CATEGORY_CASES = 5;
+/** Per-category pass rates per run, with small categories rolled into `other`. */
+export declare function buildCategoryBreakdown(rows: TestCaseComparisonRow[], runIds: string[], minCases?: number): CategoryBreakdown;
+export interface SharedWeakness {
+    category: string;
+    /** runId → pass-rate percent (0-100) in this category. */
+    rates: Record<string, number>;
+    /** Fraction (0-1) of all-fail cases that belong to this category. */
+    allFailShare: number;
+}
+/**
+ * Detect a category that is the weakest (within {@link WEAKEST_TOLERANCE_PP})
+ * for EVERY selected run — the "shared floor". When present, it usually
+ * indicates a benchmark/corpus-level problem rather than an agent choice.
+ * Returns null when runs disagree about their weakest category or nothing is
+ * genuinely weak. `other`/`uncategorized` rollups are never flagged.
+ */
+export declare function detectSharedWeakness(breakdown: CategoryBreakdown, partition: AgreementPartition, runIds: string[]): SharedWeakness | null;
+//# sourceMappingURL=comparisonInsights.d.ts.map

package/lib/dist/lib/comparisonInsights.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparisonInsights.d.ts","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,SAAS,CAAC;AAGrD,2DAA2D;AAC3D,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,SAAS,GAAG,OAAO,CAAC;AAE9D,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,6EAA6E;IAC7E,SAAS,EAAE,qBAAqB,EAAE,CAAC;CACpC;AAyBD;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,kBAAkB,CAgBpB;AAED,8FAA8F;AAC9F,wBAAgB,SAAS,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,eAAe,GAAG,IAAI,CAO9F;AAED;;;;GAIG;AACH,eAAO,MAAM,aAAa,oBAAoB,CAAC;AAC/C,wEAAwE;AACxE,eAAO,MAAM,cAAc,YAAY,CAAC;AAExC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,qBAAqB,EAAE,cAAc,GAAG,QAAQ,CAAC,GAAG,MAAM,CAQtG;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,2EAA2E;IAC3E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kGAAkG;IAClG,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC;IACrD,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;CACnC;AAED;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,kFAAkF;AAClF,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,qBAAqB,EAAE,EAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,GAAE,MAA2B,GACpC,iBAAiB,CAuCnB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,qEAAqE;IACrE,YAAY,EAAE,MAAM,CAAC;CACtB;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,iBAAiB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,MAAM,EAAE,MAAM,EAAE,GACf,cAAc,GAAG,IAAI,CAuCvB"}

package/lib/dist/lib/comparisonInsights.js

@@ -0,0 +1,212 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { getSubcategoryFromLabels } from '../lib/testCaseLabels.js';
+/** True when this run's result counts as a pass for agreement purposes. */
+function isPass(r) {
+    return r?.passFailStatus === 'passed';
+}
+/**
+ * True when the run produced *some verdict* we can bucket (pass OR fail).
+ * Three shapes count:
+ *   - a judge verdict (`passFailStatus` passed/failed);
+ *   - a run-level failure (`status: 'failed'` — the agent errored/crashed on
+ *     this case, which IS a fail verdict for agreement purposes).
+ * NOT a verdict (→ uncovered): missing results, and evaluator-errored
+ * reports (`errored: true`, `passFailStatus` cleared — issue #242 keeps
+ * "the judge broke" distinct from "the agent failed"; bucketing them as
+ * fails would poison All-fail with infrastructure noise).
+ */
+function hasVerdict(r) {
+    if (!r || r.status === 'missing')
+        return false;
+    if (r.passFailStatus === 'passed' || r.passFailStatus === 'failed')
+        return true;
+    if (r.errored)
+        return false;
+    return r.status === 'failed';
+}
+/**
+ * Partition rows into agreement buckets across the given runs.
+ * A row only participates when EVERY selected run has a verdict for it —
+ * partially-covered rows go to `uncovered` (they can't agree or disagree).
+ */
+export function partitionByAgreement(rows, runIds) {
+    const partition = { allPass: [], allFail: [], split: [], uncovered: [] };
+    if (runIds.length === 0)
+        return partition;
+    for (const row of rows) {
+        const results = runIds.map(id => row.results[id]);
+        if (results.some(r => !hasVerdict(r))) {
+            partition.uncovered.push(row);
+            continue;
+        }
+        const passes = results.filter(r => isPass(r)).length;
+        if (passes === runIds.length)
+            partition.allPass.push(row);
+        else if (passes === 0)
+            partition.allFail.push(row);
+        else
+            partition.split.push(row);
+    }
+    return partition;
+}
+/** Bucket a single row (same semantics as {@link partitionByAgreement}); null = uncovered. */
+export function bucketRow(row, runIds) {
+    const results = runIds.map(id => row.results[id]);
+    if (results.some(r => !hasVerdict(r)))
+        return null;
+    const passes = results.filter(r => isPass(r)).length;
+    if (passes === runIds.length)
+        return 'allPass';
+    if (passes === 0)
+        return 'allFail';
+    return 'split';
+}
+/**
+ * Synthetic bucket names. Parentheses are deliberately outside the
+ * name-tag charset (`[\w-]`), so a real benchmark category can never
+ * collide with these rollup buckets.
+ */
+export const UNCATEGORIZED = '(uncategorized)';
+/** Rollup bucket for categories with too few cases to be meaningful. */
+export const OTHER_CATEGORY = '(other)';
+/**
+ * Extract a row's category.
+ * Priority:
+ *   1. The `subcategory:<x>` label — the proper, purpose-built tag for this
+ *      (set via the SDK's `test(name, { labels: ['subcategory:basic'] })`,
+ *      the JSON/CLI import's `subcategory` field, or the Test Case editor).
+ *      Prefer this over anything scraped from free text: it's validated,
+ *      exported/round-tripped, and shown elsewhere in the UI (EvalsPage,
+ *      BenchmarkEditor) — a real column, not a regex guess.
+ *   2. A bracketed tag embedded in the test-case name — a convention older
+ *      imported benchmarks used before `subcategory` existed (e.g.
+ *      "qst_0011 [basic] How long is …"). Kept only for benchmarks that
+ *      predate the `subcategory` field; new imports should set it instead.
+ *   3. A `topic:<x>` label (the generic labels system, pre-dates
+ *      `subcategory` as a dedicated concept).
+ *   4. {@link UNCATEGORIZED}.
+ * (`category:<x>` labels are intentionally NOT used here: imported benchmarks
+ * stamp a single `category:RAG` on every case, which would collapse the
+ * breakdown into one column — that's the coarse domain, not this facet.)
+ */
+export function extractRowCategory(row) {
+    const subcategory = getSubcategoryFromLabels(row.labels);
+    if (subcategory)
+        return subcategory.toLowerCase();
+    const m = /\[([\w-]+)\]/.exec(row.testCaseName || '');
+    if (m)
+        return m[1].toLowerCase();
+    const topic = (row.labels || []).find(l => l.toLowerCase().startsWith('topic:'));
+    if (topic)
+        return topic.slice('topic:'.length).toLowerCase();
+    return UNCATEGORIZED;
+}
+/**
+ * Default minimum case count for a category to get its own column; smaller
+ * ones roll up into `other`. Callers with small datasets may lower this —
+ * a cell needs enough cases that a single verdict flip doesn't swing it
+ * by tens of percentage points.
+ */
+export const MIN_CATEGORY_CASES = 5;
+/** Per-category pass rates per run, with small categories rolled into `other`. */
+export function buildCategoryBreakdown(rows, runIds, minCases = MIN_CATEGORY_CASES) {
+    // Raw counts per category (row-level, for rollup decisions)
+    const rawTotals = {};
+    for (const row of rows) {
+        const cat = extractRowCategory(row);
+        rawTotals[cat] = (rawTotals[cat] || 0) + 1;
+    }
+    const keep = new Set(Object.keys(rawTotals).filter(c => rawTotals[c] >= minCases));
+    const resolve = (cat) => (keep.has(cat) ? cat : OTHER_CATEGORY);
+    const members = {};
+    for (const raw of Object.keys(rawTotals)) {
+        const col = resolve(raw);
+        (members[col] = members[col] || []).push(raw);
+    }
+    const perRun = {};
+    const totals = {};
+    for (const row of rows) {
+        const cat = resolve(extractRowCategory(row));
+        totals[cat] = (totals[cat] || 0) + 1;
+        for (const runId of runIds) {
+            const r = row.results[runId];
+            if (!hasVerdict(r))
+                continue;
+            perRun[runId] = perRun[runId] || {};
+            const cell = (perRun[runId][cat] = perRun[runId][cat] || { passed: 0, total: 0 });
+            cell.total += 1;
+            if (isPass(r))
+                cell.passed += 1;
+        }
+    }
+    const categories = Object.keys(totals).sort((a, b) => {
+        if (a === OTHER_CATEGORY)
+            return 1;
+        if (b === OTHER_CATEGORY)
+            return -1;
+        return (totals[b] || 0) - (totals[a] || 0);
+    });
+    return { categories, perRun, totals, members };
+}
+/**
+ * Tie tolerance (pp) for the "weakest" claim. Kept at rounding-error level
+ * on purpose: the callout says "weakest category", so another category may
+ * not be meaningfully lower for any run — a 5pp allowance here would make
+ * the copy dishonest.
+ */
+const WEAKEST_TOLERANCE_PP = 1;
+/** Don't flag a shared weakness unless the mean rate is actually weak. */
+const WEAKNESS_MAX_MEAN_RATE = 75;
+/**
+ * Detect a category that is the weakest (within {@link WEAKEST_TOLERANCE_PP})
+ * for EVERY selected run — the "shared floor". When present, it usually
+ * indicates a benchmark/corpus-level problem rather than an agent choice.
+ * Returns null when runs disagree about their weakest category or nothing is
+ * genuinely weak. `other`/`uncategorized` rollups are never flagged.
+ */
+export function detectSharedWeakness(breakdown, partition, runIds) {
+    const candidates = breakdown.categories.filter(c => c !== OTHER_CATEGORY && c !== UNCATEGORIZED);
+    if (candidates.length < 2 || runIds.length === 0)
+        return null;
+    const rate = (runId, cat) => {
+        const cell = breakdown.perRun[runId]?.[cat];
+        if (!cell || cell.total === 0)
+            return null;
+        return (cell.passed / cell.total) * 100;
+    };
+    // Mean-weakest candidate across runs.
+    let best = null;
+    for (const cat of candidates) {
+        const rates = runIds.map(id => rate(id, cat)).filter((x) => x !== null);
+        if (rates.length !== runIds.length)
+            continue; // needs coverage in every run
+        const mean = rates.reduce((a, b) => a + b, 0) / rates.length;
+        if (!best || mean < best.mean)
+            best = { cat, mean };
+    }
+    if (!best || best.mean >= WEAKNESS_MAX_MEAN_RATE)
+        return null;
+    // It must be every run's weakest category (within tolerance).
+    for (const runId of runIds) {
+        const candidateRate = rate(runId, best.cat);
+        if (candidateRate === null)
+            return null;
+        for (const cat of candidates) {
+            if (cat === best.cat)
+                continue;
+            const r = rate(runId, cat);
+            if (r !== null && r < candidateRate - WEAKEST_TOLERANCE_PP)
+                return null;
+        }
+    }
+    const rates = {};
+    for (const runId of runIds)
+        rates[runId] = Math.round(rate(runId, best.cat));
+    const allFailInCat = partition.allFail.filter(row => extractRowCategory(row) === best.cat).length;
+    const allFailShare = partition.allFail.length > 0 ? allFailInCat / partition.allFail.length : 0;
+    return { category: best.cat, rates, allFailShare };
+}
+//# sourceMappingURL=comparisonInsights.js.map

package/lib/dist/lib/comparisonInsights.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparisonInsights.js","sourceRoot":"","sources":["../../comparisonInsights.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAiBH,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAahE,2EAA2E;AAC3E,SAAS,MAAM,CAAC,CAAiD;IAC/D,OAAO,CAAC,EAAE,cAAc,KAAK,QAAQ,CAAC;AACxC,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,UAAU,CAAC,CAAqF;IACvG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAChF,IAAI,CAAC,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAC5B,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,IAA6B,EAC7B,MAAgB;IAEhB,MAAM,SAAS,GAAuB,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC7F,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;QAClD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACrD,IAAI,MAAM,KAAK,CAAC;YAAE,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YAC9C,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,SAAS,CAAC,GAA0B,EAAE,MAAgB;IACpE,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC/C,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACnC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,iBAAiB,CAAC;AAC/C,wEAAwE;AACxE,MAAM,CAAC,MAAM,cAAc,GAAG,SAAS,CAAC;AAExC;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAA2D;IAC5F,MAAM,WAAW,GAAG,wBAAwB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzD,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC,WAAW,EAAE,CAAC;IAClD,MAAM,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACtD,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;IACjF,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC7D,OAAO,aAAa,CAAC;AACvB,CAAC;AAuBD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAEpC,kFAAkF;AAClF,MAAM,UAAU,sBAAsB,CACpC,IAA6B,EAC7B,MAAgB,EAChB,WAAmB,kBAAkB;IAErC,4DAA4D;IAC5D,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACpC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC;IACnF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;IAExE,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,MAAM,GAAiD,EAAE,CAAC;IAChE,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC7B,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC7B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;YAClF,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;YAChB,IAAI,MAAM,CAAC,CAAC,CAAC;gBAAE,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACnD,IAAI,CAAC,KAAK,cAAc;YAAE,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,cAAc;YAAE,OAAO,CAAC,CAAC,CAAC;QACpC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACjD,CAAC;AAUD;;;;;GAKG;AACH,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,0EAA0E;AAC1E,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,SAA4B,EAC5B,SAA6B,EAC7B,MAAgB;IAEhB,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAC5C,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,cAAc,IAAI,CAAC,KAAK,aAAa,CACjD,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9D,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,GAAW,EAAiB,EAAE;QACzD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC3C,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;IAC1C,CAAC,CAAC;IAEF,sCAAsC;IACtC,IAAI,IAAI,GAAyC,IAAI,CAAC;IACtD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QACrF,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,SAAS,CAAC,8BAA8B;QAC5E,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QAC7D,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI;YAAE,IAAI,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,sBAAsB;QAAE,OAAO,IAAI,CAAC;IAE9D,8DAA8D;IAC9D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,aAAa,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,GAAG,KAAK,IAAI,CAAC,GAAG;gBAAE,SAAS;YAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,aAAa,GAAG,oBAAoB;gBAAE,OAAO,IAAI,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAE,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,kBAAkB,CAAC,GAAG,CAAC,KAAK,IAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnG,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhG,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AACrD,CAAC"}

package/lib/dist/lib/config/defineConfig.d.ts

@@ -0,0 +1,27 @@
+/**
+ * defineConfig Helper
+ * Provides type safety for agent-health.config.ts files
+ */
+import type { UserConfig } from './types.js';
+/**
+ * Define configuration with full type safety
+ *
+ * @example
+ * ```typescript
+ * // agent-health.config.ts
+ * import { defineConfig } from '@opensearch-project/agent-health';
+ *
+ * export default defineConfig({
+ *   agents: [
+ *     {
+ *       key: 'my-agent',
+ *       name: 'My Agent',
+ *       endpoint: 'https://api.example.com/chat',
+ *       connectorType: 'rest',
+ *     },
+ *   ],
+ * });
+ * ```
+ */
+export declare function defineConfig(config: UserConfig): UserConfig;
+//# sourceMappingURL=defineConfig.d.ts.map

package/lib/dist/lib/config/defineConfig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"defineConfig.d.ts","sourceRoot":"","sources":["../../../config/defineConfig.ts"],"names":[],"mappings":"AAKA;;;GAGG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE1C;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAE3D"}

package/lib/dist/lib/config/defineConfig.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Define configuration with full type safety
+ *
+ * @example
+ * ```typescript
+ * // agent-health.config.ts
+ * import { defineConfig } from '@opensearch-project/agent-health';
+ *
+ * export default defineConfig({
+ *   agents: [
+ *     {
+ *       key: 'my-agent',
+ *       name: 'My Agent',
+ *       endpoint: 'https://api.example.com/chat',
+ *       connectorType: 'rest',
+ *     },
+ *   ],
+ * });
+ * ```
+ */
+export function defineConfig(config) {
+    return config;
+}
+//# sourceMappingURL=defineConfig.js.map

package/lib/dist/lib/config/defineConfig.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"defineConfig.js","sourceRoot":"","sources":["../../../config/defineConfig.ts"],"names":[],"mappings":"AAAA;;;GAGG;AASH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/lib/dist/lib/config/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Config Module
+ * Public exports for configuration system
+ */
+export type { UserConfig, UserAgentConfig, UserModelConfig, ResolvedConfig, ConfigFileInfo, ReporterConfig, JudgeConfig, ServerConfig, ResolvedServerConfig, } from './types.js';
+export type { AgentHooks, BeforeRequestContext } from '../../types/index.js';
+export { defineConfig } from './defineConfig.js';
+export { loadConfig, loadConfigSync, findConfigFile, getConfigFileInfo, clearConfigCache, DEFAULT_SERVER_CONFIG, } from './loader.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/dist/lib/config/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../config/index.ts"],"names":[],"mappings":"AAKA;;;GAGG;AAGH,YAAY,EACV,UAAU,EACV,eAAe,EACf,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,WAAW,EACX,YAAY,EACZ,oBAAoB,GACrB,MAAM,SAAS,CAAC;AAGjB,YAAY,EAAE,UAAU,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAGhE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,UAAU,EACV,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,UAAU,CAAC"}

package/lib/dist/lib/config/index.js

@@ -0,0 +1,8 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+// Function exports
+export { defineConfig } from './defineConfig.js';
+export { loadConfig, loadConfigSync, findConfigFile, getConfigFileInfo, clearConfigCache, DEFAULT_SERVER_CONFIG, } from './loader.js';
+//# sourceMappingURL=index.js.map

package/lib/dist/lib/config/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../config/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,mBAAmB;AACnB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,UAAU,EACV,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,GACtB,MAAM,UAAU,CAAC"}

package/lib/dist/lib/config/loader.d.ts

@@ -0,0 +1,39 @@
+import type { ResolvedConfig, ResolvedServerConfig, ConfigFileInfo } from './types.js';
+/**
+ * Default server configuration
+ * Follows Playwright's webServer pattern
+ */
+export declare const DEFAULT_SERVER_CONFIG: ResolvedServerConfig;
+/**
+ * Runtime state file (config v2: `.agent-health/state.json`, user or project
+ * scope). Loaded by the *server services* (configService, customAgentStore,
+ * dataSourceConfig), not by this loader. We only probe for it so the startup
+ * log can tell the user whether on-disk runtime state is in effect (ui-first
+ * mode) instead of saying "No config file found".
+ */
+export declare function hasServerJsonConfig(cwd?: string): boolean;
+/**
+ * Find config file in the given directory
+ */
+export declare function findConfigFile(cwd?: string): ConfigFileInfo | null;
+/**
+ * Load and resolve configuration
+ *
+ * @param cwd - Working directory to search for config file
+ * @param force - Force reload even if cached
+ */
+export declare function loadConfig(cwd?: string, force?: boolean): Promise<ResolvedConfig>;
+/**
+ * Load config synchronously (for use in module initialization)
+ * Returns defaults if no config file or if loading fails
+ */
+export declare function loadConfigSync(cwd?: string): ResolvedConfig;
+/**
+ * Clear config cache (useful for testing)
+ */
+export declare function clearConfigCache(): void;
+/**
+ * Get config file info without loading
+ */
+export declare function getConfigFileInfo(cwd?: string): ConfigFileInfo | null;
+//# sourceMappingURL=loader.d.ts.map

package/lib/dist/lib/config/loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../config/loader.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAIV,cAAc,EACd,oBAAoB,EACpB,cAAc,EAIf,MAAM,SAAS,CAAC;AAEjB;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,oBAInC,CAAC;AAWF;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,GAAE,MAAsB,GAAG,OAAO,CAExE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAsB,GAAG,cAAc,GAAG,IAAI,CASjF;AAwJD;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,GAAG,GAAE,MAAsB,EAC3B,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,cAAc,CAAC,CAqCzB;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAsB,GAAG,cAAc,CAmB1E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAGvC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,GAAE,MAAsB,GAAG,cAAc,GAAG,IAAI,CAEpF"}