npm - @opensearch-project/agent-health - Versions diffs - 0.5.2 → 0.6.0 - Mend

@opensearch-project/agent-health 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (126) hide show

package/README.md +1 -0
package/cli/dist/index.js +1592 -620
package/dist/assets/index-BfxtxmKc.css +1 -0
package/dist/assets/index-CrjAfDHu.js +243 -0
package/dist/index.html +2 -2
package/docs/CLI.md +45 -2
package/docs/CONFIGURATION.md +1 -1
package/docs/INSTRUMENT_WITH_OTEL.md +7 -0
package/docs/SDK.md +52 -3
package/docs/STORAGE_INDEX_FIELD_LIMITS.md +216 -0
package/docs/skills/AGENT_HEALTH.md +55 -1
package/examples/eval-files/demo.eval.js +1 -1
package/examples/eval-files/ops-rca-classification.eval.js +71 -0
package/examples/eval-files/ops-rca-evaluator.json +15 -0
package/examples/eval-files/sdk-demo.eval.js +72 -0
package/examples/eval-files/sdk-describe-demo.eval.js +50 -0
package/examples/eval-files/sdk-hooks-demo.eval.js +1 -1
package/lib/dist/lib/bedrockCompat.d.ts +27 -0
package/lib/dist/lib/bedrockCompat.d.ts.map +1 -0
package/lib/dist/lib/bedrockCompat.js +83 -0
package/lib/dist/lib/bedrockCompat.js.map +1 -0
package/lib/dist/lib/benchmarkImage.d.ts +52 -0
package/lib/dist/lib/benchmarkImage.d.ts.map +1 -0
package/lib/dist/lib/benchmarkImage.js +113 -0
package/lib/dist/lib/benchmarkImage.js.map +1 -0
package/lib/dist/lib/benchmarkVersionUtils.d.ts +13 -0
package/lib/dist/lib/benchmarkVersionUtils.d.ts.map +1 -1
package/lib/dist/lib/benchmarkVersionUtils.js +21 -0
package/lib/dist/lib/benchmarkVersionUtils.js.map +1 -1
package/lib/dist/lib/chunkedFetch.d.ts +18 -0
package/lib/dist/lib/chunkedFetch.d.ts.map +1 -0
package/lib/dist/lib/chunkedFetch.js +40 -0
package/lib/dist/lib/chunkedFetch.js.map +1 -0
package/lib/dist/lib/comparisonInsights.d.ts +104 -0
package/lib/dist/lib/comparisonInsights.d.ts.map +1 -0
package/lib/dist/lib/comparisonInsights.js +212 -0
package/lib/dist/lib/comparisonInsights.js.map +1 -0
package/lib/dist/lib/config/loader.d.ts.map +1 -1
package/lib/dist/lib/config/loader.js +5 -0
package/lib/dist/lib/config/loader.js.map +1 -1
package/lib/dist/lib/config/types.d.ts +14 -0
package/lib/dist/lib/config/types.d.ts.map +1 -1
package/lib/dist/lib/constants.d.ts +11 -0
package/lib/dist/lib/constants.d.ts.map +1 -1
package/lib/dist/lib/constants.js +10 -1
package/lib/dist/lib/constants.js.map +1 -1
package/lib/dist/lib/contextFormat.d.ts +26 -0
package/lib/dist/lib/contextFormat.d.ts.map +1 -0
package/lib/dist/lib/contextFormat.js +28 -0
package/lib/dist/lib/contextFormat.js.map +1 -0
package/lib/dist/lib/envCompat.d.ts.map +1 -1
package/lib/dist/lib/envCompat.js +14 -5
package/lib/dist/lib/envCompat.js.map +1 -1
package/lib/dist/lib/evaluationRerun.d.ts +63 -0
package/lib/dist/lib/evaluationRerun.d.ts.map +1 -0
package/lib/dist/lib/evaluationRerun.js +85 -0
package/lib/dist/lib/evaluationRerun.js.map +1 -0
package/lib/dist/lib/matchers/traces.d.ts +17 -2
package/lib/dist/lib/matchers/traces.d.ts.map +1 -1
package/lib/dist/lib/matchers/traces.js +136 -16
package/lib/dist/lib/matchers/traces.js.map +1 -1
package/lib/dist/lib/matchers/tracesPricing.d.ts +38 -0
package/lib/dist/lib/matchers/tracesPricing.d.ts.map +1 -0
package/lib/dist/lib/matchers/tracesPricing.js +64 -0
package/lib/dist/lib/matchers/tracesPricing.js.map +1 -0
package/lib/dist/lib/runStats.d.ts +24 -0
package/lib/dist/lib/runStats.d.ts.map +1 -1
package/lib/dist/lib/runStats.js +32 -0
package/lib/dist/lib/runStats.js.map +1 -1
package/lib/dist/lib/testCases/loader.d.ts +11 -1
package/lib/dist/lib/testCases/loader.d.ts.map +1 -1
package/lib/dist/lib/testCases/loader.js +35 -5
package/lib/dist/lib/testCases/loader.js.map +1 -1
package/lib/dist/lib/utils.d.ts +15 -0
package/lib/dist/lib/utils.d.ts.map +1 -1
package/lib/dist/lib/utils.js +22 -0
package/lib/dist/lib/utils.js.map +1 -1
package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts +10 -0
package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.d.ts.map +1 -1
package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js +17 -3
package/lib/dist/services/connectors/claude-code/ClaudeCodeConnector.js.map +1 -1
package/lib/dist/services/connectors/subprocess/SubprocessConnector.d.ts.map +1 -1
package/lib/dist/services/connectors/subprocess/SubprocessConnector.js +8 -0
package/lib/dist/services/connectors/subprocess/SubprocessConnector.js.map +1 -1
package/lib/dist/services/evaluation/bedrockJudge.d.ts +7 -0
package/lib/dist/services/evaluation/bedrockJudge.d.ts.map +1 -1
package/lib/dist/services/evaluation/bedrockJudge.js +2 -0
package/lib/dist/services/evaluation/bedrockJudge.js.map +1 -1
package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts +1 -1
package/lib/dist/services/storage/asyncBenchmarkStorage.d.ts.map +1 -1
package/lib/dist/services/storage/asyncBenchmarkStorage.js +14 -3
package/lib/dist/services/storage/asyncBenchmarkStorage.js.map +1 -1
package/lib/dist/services/storage/asyncRunStorage.d.ts +9 -1
package/lib/dist/services/storage/asyncRunStorage.d.ts.map +1 -1
package/lib/dist/services/storage/asyncRunStorage.js +77 -1
package/lib/dist/services/storage/asyncRunStorage.js.map +1 -1
package/lib/dist/services/storage/asyncTestCaseStorage.d.ts +18 -4
package/lib/dist/services/storage/asyncTestCaseStorage.d.ts.map +1 -1
package/lib/dist/services/storage/asyncTestCaseStorage.js +20 -4
package/lib/dist/services/storage/asyncTestCaseStorage.js.map +1 -1
package/lib/dist/services/storage/opensearchClient.d.ts +35 -12
package/lib/dist/services/storage/opensearchClient.d.ts.map +1 -1
package/lib/dist/services/storage/opensearchClient.js +12 -5
package/lib/dist/services/storage/opensearchClient.js.map +1 -1
package/lib/dist/services/traces/browserRecovery.d.ts +12 -1
package/lib/dist/services/traces/browserRecovery.d.ts.map +1 -1
package/lib/dist/services/traces/browserRecovery.js +32 -5
package/lib/dist/services/traces/browserRecovery.js.map +1 -1
package/lib/dist/services/traces/messageExtraction.d.ts.map +1 -1
package/lib/dist/services/traces/messageExtraction.js +95 -33
package/lib/dist/services/traces/messageExtraction.js.map +1 -1
package/lib/dist/services/traces/spansToTrajectory.d.ts.map +1 -1
package/lib/dist/services/traces/spansToTrajectory.js +47 -9
package/lib/dist/services/traces/spansToTrajectory.js.map +1 -1
package/lib/dist/services/traces/tracePoller.d.ts +27 -3
package/lib/dist/services/traces/tracePoller.d.ts.map +1 -1
package/lib/dist/services/traces/tracePoller.js +199 -33
package/lib/dist/services/traces/tracePoller.js.map +1 -1
package/lib/dist/types/index.d.ts +55 -1
package/lib/dist/types/index.d.ts.map +1 -1
package/lib/dist/types/index.js.map +1 -1
package/package.json +3 -1
package/server/dist/app.js +2238 -777
package/server/dist/index.js +2238 -777
package/dist/assets/index-CCQRDlO0.js +0 -243
package/dist/assets/index-CNHQVbcj.css +0 -1

package/dist/index.html CHANGED Viewed

@@ -13,8 +13,8 @@
     <link rel="preconnect" href="https://fonts.googleapis.com">
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
     <link href="https://fonts.googleapis.com/css2?family=Rubik:wght@300;400;500;600;700&family=Source+Code+Pro:wght@400;500;600&display=swap" rel="stylesheet">
-    <script type="module" crossorigin src="/assets/index-CCQRDlO0.js"></script>
-    <link rel="stylesheet" crossorigin href="/assets/index-CNHQVbcj.css">
+    <script type="module" crossorigin src="/assets/index-CrjAfDHu.js"></script>
+    <link rel="stylesheet" crossorigin href="/assets/index-BfxtxmKc.css">
   </head>
   <body class="bg-background text-foreground antialiased h-full">
     <div id="root" class="h-full"></div>

package/docs/CLI.md CHANGED Viewed

@@ -61,11 +61,13 @@ agent-health list <resource> [-o table|json]
 | `models` | | Available models |
 | `test-cases` | `testcases`, `tc` | Stored test cases |
 | `benchmarks` | `bench` | Stored benchmarks |
+| `images` | `img` | Benchmark images (content-addressed evaluation-condition snapshots; runs sharing a digest are directly comparable) |
 ```bash
 agent-health list agents
 agent-health list tc -o json
 agent-health list bench
+agent-health list images
 ```
 ---
@@ -122,20 +124,61 @@ agent-health benchmark [options]
 | `--stop-server` | Stop the server after benchmark completes | Keep running |
 **Modes:**
-- **Quick mode** (no `-n`, no `-f`): Auto-creates a benchmark from all stored test cases
+- **Quick mode** (no `-n`, no `-f`): Runs all stored test cases as an **ad-hoc evaluation run** (no benchmark entity is created)
 - **Named mode** (`-n <name>`): Runs a specific existing benchmark
 - **File mode** (`-f <path>`): Imports test cases from a JSON file **or runs a code SDK file** (`.eval.js` / `.eval.ts` — see [SDK.md](./SDK.md)), creates a benchmark, and runs it
+Every evaluation run is stamped with an **image digest** — a content hash of
+its test-case contents + eval conditions (evaluator, judge model). Runs with
+the same digest ran under identical conditions and are directly comparable;
+re-running the same command converges on the same image instead of creating
+new entities. See `agent-health list images`.
 ```bash
 agent-health benchmark                                           # quick mode
 agent-health benchmark -n "Baseline" -a ml-commons               # named mode
 agent-health benchmark -f ./test-cases.json -a pulsar -v         # file mode (JSON)
-agent-health benchmark -f ./evals/demo.eval.js -a observio       # file mode (code SDK)
+agent-health benchmark -f ./examples/eval-files/demo.eval.js -a observio       # file mode (code SDK)
 agent-health benchmark -f ./test-cases.json -n "My Run" -a pulsar --export results.json
 agent-health benchmark -n "Baseline" -e system-tool-usage -c 4   # custom evaluator, 4 in parallel
 agent-health benchmark -n "Baseline" --export report.html --format html
 ```
+#### benchmark doctor
+Detect and clean up duplicated / debris benchmarks (dry-run by default).
+```
+agent-health benchmark doctor [--dry-run] [--apply] [--migrate-images] [--json]
+```
+| Option | Description |
+|--------|-------------|
+| `--dry-run` | Preview only — this is already the default; use --apply to execute |
+| `--apply` | Execute the plan (default: dry-run report only) |
+| `--migrate-images` | Convert remaining benchmarks into tagged benchmark images |
+| `--json` | Output as JSON instead of the human-readable report |
+What it detects:
+- **Timestamped debris** — `quick-<ts>` / `*-<epoch-ms>` benchmarks with no
+  runs anywhere and older than 24h (deleted).
+- **Content duplicates** — benchmarks with identical test-case sets. Embedded
+  runs are merged into the canonical (most runs › most references › oldest),
+  evaluation runs are re-pointed, duplicate shells deleted.
+Runs and reports are **never** deleted. Sample data (`demo-*`) is never touched.
+**Read-only mode**: When running without `--apply` or `--migrate-images` (pure dry-run),
+the command safely reuses existing foreign servers in read-only mode. This allows
+the diagnostic to run even when another agent-health instance is operating on a
+different worktree/port, with a clear notice that no writes will be issued.
+```bash
+agent-health benchmark doctor                    # report what would change (read-only)
+agent-health benchmark doctor --apply            # clean up (strict server guard)
+agent-health benchmark doctor --apply --migrate-images
+```
 ---
 ### export

package/docs/CONFIGURATION.md CHANGED Viewed

@@ -113,7 +113,7 @@ Rules of thumb:
   `OPENSEARCH_STORAGE_*` or `storage` in your config to use a cluster instead).
 - **`evals/`** is **your test source** — `.eval.js` / `.eval.ts` files you write
   with the [code SDK](./SDK.md) and run with
-  `agent-health benchmark -f ./evals/demo.eval.js`. They are **not** stored under
+  `agent-health benchmark -f ./examples/eval-files/demo.eval.js`. They are **not** stored under
   `.agent-health/data/`; *running* them produces run records that land there (or
   in OpenSearch).

package/docs/INSTRUMENT_WITH_OTEL.md CHANGED Viewed

@@ -385,6 +385,13 @@ If you run an MCP server, expose this document as a resource at `agent-health://
 ## Reference
+### New to OpenTelemetry?
+- [What is OpenTelemetry?](https://opentelemetry.io/docs/what-is-opentelemetry/) — start here for the concepts (traces, spans, exporters, collectors).
+- [OpenTelemetry GenAI semantic conventions (repo)](https://github.com/open-telemetry/semantic-conventions-genai) — the source-of-truth for the `gen_ai.*` attributes this guide uses.
+### Specs used by this guide
 - [OTel GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
 - [OTel GenAI Agent Spans](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/)
 - [Agent Health Telemetry Setup](./CLAUDE_CODE_TELEMETRY.md)

package/docs/SDK.md CHANGED Viewed

@@ -166,7 +166,7 @@ Key points:
 Hooks are a no-op when no test in the run uses them — the orchestrator
 is short-circuited to a noop variant and existing tests pay zero cost.
-See the demo at [`evals/sdk-hooks-demo.eval.js`](../evals/sdk-hooks-demo.eval.js).
+See the demo at [`examples/eval-files/sdk-hooks-demo.eval.js`](../examples/eval-files/sdk-hooks-demo.eval.js).
 ### 5. Matchers record structured verdicts
@@ -483,8 +483,57 @@ enforced regardless of configuration.
 ---
+## Weighted scoring across criteria
+Each `expect(...)`, `judge(...)`, and `evaluate(...)` call records **one
+per-matcher pass/fail** — they are not weighted against each other. When you
+need a **single aggregate score** that weights criteria differently (e.g. root
+cause 60%, SOP selection 20%, relevant-metrics check 10%, latency 10%), the
+weighting lives in a **custom evaluator's `scoringConfig`**, not in the test
+body. Define the weighted metrics once and attach the evaluator to the run via
+`evaluatorId`:
+```json
+{
+  "name": "Ops RCA (weighted)",
+  "systemPrompt": "You evaluate an ops agent that triages a ticket. Score each metric 0-100. CRITICAL: root_cause_accuracy is the primary metric.",
+  "scoringConfig": {
+    "metrics": [
+      { "name": "root_cause_accuracy",  "weight": 0.6, "scale": 100 },
+      { "name": "sop_selection",        "weight": 0.2, "scale": 100 },
+      { "name": "relevant_metrics",     "weight": 0.1, "scale": 100 },
+      { "name": "latency",              "weight": 0.1, "scale": 100 }
+    ],
+    "passThreshold": 80,
+    "scale": 100
+  }
+}
+```
+The run's overall score is the weighted mean of the metrics the evaluator
+emits, so you get one comparable number per run for progress tracking. Use the
+deterministic `expect(...)` matchers for the hard checks (exact classification,
+tool was called, budget cap) and the judge/evaluator + weighted `scoringConfig`
+for the aggregate score. See the ops-RCA worked example at
+[`examples/eval-files/ops-rca-classification.eval.js`](../examples/eval-files/ops-rca-classification.eval.js)
+and its companion [`examples/eval-files/ops-rca-evaluator.json`](../examples/eval-files/ops-rca-evaluator.json),
+and the evaluator reference in [docs/skills/AGENT_HEALTH.md](./skills/AGENT_HEALTH.md#custom-evaluators).
+---
 ## Running the tests
+> **Security note:** the ENTIRE contents of an imported `.eval.js`/`.eval.ts`
+> file are persisted verbatim on the resulting test case (`sourceCode`) and
+> rendered on the Test Case detail page as an IDE-style code view, so anyone
+> who can view test cases in this deployment can read the full file --
+> including any comments, hardcoded values, or internal URLs it contains.
+> Treat eval files like any other source file that lands in your repo and
+> gets deployed alongside the app: don't hardcode secrets, tokens, or
+> customer data in them -- pull those from environment variables /
+> `agent-health.config.ts` instead, same as you already would for the app's
+> own credentials.
 ### Via the UI
 `/evaluations/runs/new` → pick "Code import" → select your `.eval.js` files.
@@ -492,7 +541,7 @@ enforced regardless of configuration.
 ### Via the CLI
 ```bash
-npx @opensearch-project/agent-health benchmark -f ./evals/demo.eval.js -a observio
+npx @opensearch-project/agent-health benchmark -f ./examples/eval-files/demo.eval.js -a observio
 ```
 ### Via the HTTP API
@@ -504,7 +553,7 @@ curl -sN -X POST http://localhost:4001/api/storage/evaluation-runs \
     "name": "Demo",
     "sources": [{
       "type": "code-import",
-      "filenames": ["evals/demo.eval.js"],
+      "filenames": ["examples/eval-files/demo.eval.js"],
       "testCaseIds": []
     }],
     "agentKey": "observio",

package/docs/STORAGE_INDEX_FIELD_LIMITS.md ADDED Viewed

@@ -0,0 +1,216 @@
+# Storage: OpenSearch index field-limit growth (`evals_runs`)
+## Incident
+Owner-hit while running code-QA benchmarks: report/run persistence on the
+shared cluster's `evals_runs` index failed with
+```
+illegal_argument_exception: Limit of total fields [5000] has been exceeded
+```
+Run **execution** succeeded — only the **write** errored out, i.e. data loss
+(the completed report was never persisted).
+This is the same *class* of bug PR #418 fixed for `evals_experiments`
+(`EvaluationRun.results` / `testCaseSnapshots`): OpenSearch's default dynamic
+mapping mints a new mapped field for every previously-unseen key under a
+free-form object, and the field-count budget (`index.mapping.total_fields.limit`)
+is **shared across every document in the index**, not per-document. #418 did
+not cover `evals_runs` (the report/`TestCaseRun` index used by the code-SDK
+path) — this fix does.
+## Root cause: two unprotected growth vectors in `evals_runs`
+Both are driven by the same source: `EvaluationMetrics`
+(`types/index.ts`) is an open index signature (`[key: string]: number |
+undefined`) by design — custom/system evaluators declare arbitrary metric
+dimension names via `evaluator.scoringConfig.metrics`
+(`server/services/judgeResponseParser.ts`'s `extractMetrics()`,
+`services/storage/asyncRunStorage.ts`'s `storedMetricsToApp()` /
+`toStorageFormat()` — see the comments in both, which explicitly call out
+"preserve every metric the judge emitted, not just the four legacy keys").
+Every *distinct* custom metric name, across every run/matcher ever written,
+used to mint a brand-new mapped field, shared index-wide, forever.
+| Field (in `evals_runs`) | Shape | Growth vector |
+|---|---|---|
+| `metrics` (report-level) | `Record<string, number>` | One set of dynamic names per run — one custom evaluator with N metric names adds ≤N new fields **the first time it's seen**, but a code-QA benchmark suite iterating on many custom evaluators over time accumulates without bound. |
+| `matcherResults[].judgeMetrics` | `Record<string, number>`, nested inside a `nested`-typed array | Same growth, but **per SDK `judge()` call** — a single code-QA test case with many `expect`/`judge()` claims × many custom judge dimensions multiplies fast. This is the "code-SDK path" referenced in the incident — `matcherResults` is populated exclusively by the code-based test SDK (`docs/SDK.md`), not the legacy UI-driven runner. |
+Everything else already flagged in the original bug report — matcher
+`actual`/`expected`, `trajectory`, `logs`, `rawEvents`, `improvementStrategies`,
+`spans` (span attributes) — was **already** `{ type: 'object', enabled: false
+}` in `server/constants/indexMappings.ts` before this change (audited, not
+touched). `llmJudgeResponse` (which itself has an open `extraFields`/
+`parsedMetrics` shape) is **never persisted** to `evals_runs` at all
+(`toStorageFormat()` doesn't include it) — confirmed via `git grep
+llmJudgeResponse services/storage server/adapters`, no hits — so it isn't a
+growth vector for this index either.
+## Fix (mirrors #418's pattern)
+`server/constants/indexMappings.ts`, `evals_runs` index:
+```diff
+   metrics: {
++    dynamic: false,
+     properties: {
+       accuracy: { type: 'float' },
+       faithfulness: { type: 'float' },
+       latency_score: { type: 'float' },
+       trajectory_alignment_score: { type: 'float' },
+     },
+   },
+   ...
+   matcherResults: {
+     type: 'nested',
+     properties: {
+       ...
+       judgeMetrics: {
++        dynamic: false,
+         properties: {
+           accuracy: { type: 'float' },
+           faithfulness: { type: 'float' },
+           latency_score: { type: 'float' },
+           trajectory_alignment_score: { type: 'float' },
+         },
+       },
+     },
+   },
+```
+Unlike #418's `results`/`testCaseSnapshots` (`enabled: false`, fully opaque),
+this uses `dynamic: false` **with explicit typed sub-properties** for the
+four legacy metric names — they stay real, typed, queryable fields (nothing
+queries them today — see the audit below — but it's free to keep them typed),
+while every *other* metric/dimension name is stored in `_source` (readable,
+unaffected) but never added to the mapping. `_source` is unaffected either
+way — the choice between `enabled:false` and `dynamic:false` only changes
+what OpenSearch can filter/sort/aggregate on, never what's persisted or
+returned.
+## Query audit — nothing queried becomes unsearchable
+Every OpenSearch-level query/filter/sort/aggregation against `evals_runs`
+(`server/adapters/opensearch/StorageModule.ts`'s `OpenSearchRunOperations`)
+was enumerated. None touch `metrics.*` or `matcherResults[].judgeMetrics.*`
+beyond the four legacy names, which stay mapped:
+| Consumer | Query | Fields used | Affected by this fix? |
+|---|---|---|---|
+| `OpenSearchRunOperations.search()` | `term` filters | `experimentId`, `experimentRunId`, `testCaseId`, `agentId`, `modelId`, `status`, `passFailStatus` | No — untouched, still explicit `keyword` fields |
+| `OpenSearchRunOperations.search()` | `range` filter | `createdAt` | No — untouched, still `date` |
+| `OpenSearchRunOperations.getAll()` / `.search()` | `sort` | `createdAt` | No |
+| `OpenSearchRunOperations.countsByTestCase()` | `terms` agg | `testCaseId` | No |
+| `asyncRunStorage.ts` `SearchQuery.minAccuracy` | **application-level** `Array.filter()`, not an OpenSearch query (`reports.filter(r => r.metrics.accuracy >= ...)`) | `metrics.accuracy` (read from `_source` in JS) | No — reads the value out of `_source`, which is unaffected by `dynamic: false`. If this were ever converted to a server-side `range` query, it would still work: `accuracy` stays an explicitly mapped, queryable field. |
+| UI (`MatcherResultsPanel.tsx`, `JudgeSection.tsx`, `RunDetailsContent.tsx`) | none — reads `matcherResults`/`judgeMetrics` out of the fetched JSON document, never issues its own OpenSearch query | n/a | No |
+| `services/evaluation/index.ts`, `services/benchmarkRunner.ts`, `services/hookOrchestrator.ts` | none — same, in-process consumption of the already-fetched report | n/a | No |
+Conclusion: **no consumer anywhere issues an OpenSearch-side query against a
+non-legacy `metrics.*` or `judgeMetrics.*` name.** Both are read back via
+`_source` wherever consumed (search, list, comparison, UI). This mirrors
+exactly the trade-off #418 already made and documented for
+`EvaluationRun.results`.
+## Migration story — what to run, exactly
+**Nothing runs automatically against the live cluster from this PR.**
+### New / fresh indexes
+No action needed. `ensureIndexes()` (`server/services/indexInitializer.ts`,
+called on every server boot and on "attach new cluster") creates any missing
+index straight from the updated `INDEX_MAPPINGS` — new deployments and any
+environment that doesn't have `evals_runs` yet get the fix immediately.
+### Existing, NOT-YET-poisoned `evals_runs` (most environments)
+Also no action needed, but not immediate — `ensureIndexes()` also calls
+`client.indices.putMapping()` on every boot for existing indexes, which is
+how the `dynamic: false` fix reaches an already-existing-but-clean index: it
+succeeds silently and the index is protected from the next write onward.
+### The shared cluster's `evals_runs`, if already poisoned
+If any code-QA benchmark run already wrote a custom evaluator metric name to
+the shared cluster before this fix ships, `evals_runs.metrics` (and/or
+`matcherResults.judgeMetrics`) already has real, dynamically-inferred
+sub-properties. OpenSearch's `putMapping` **rejects** an `enabled`/`dynamic`
+change on a field that already has sub-properties
+(`mapper_exception: the [dynamic] parameter can't be updated for the object
+mapping [metrics]`) — `ensureIndexes()` catches this, logs a warning, and
+otherwise no-ops (no crash, no data loss, same as #418's documented
+`mapper_exception` handling for `results`). **The index keeps growing** until
+an explicit reindex is run.
+**This is not new migration code** — the existing generic reindex mechanism
+(`reindexSingleIndex()`, `server/services/mappingFixer.ts`, already shipped
+and already exposed at `POST /api/storage/reindex`, `server/routes/storage/admin.ts`)
+already recreates any `INDEX_MAPPINGS`-registered index from scratch and
+copies every document across, which sheds a poisoned mapping's dynamically-
+inferred sub-fields while preserving 100% of the underlying `_source` data
+(proven in `tests/integration/services/storage/evalsRunsMappingMigrationRecipe.integration.test.ts`,
+run against a real OpenSearch container with a deliberately-poisoned index).
+**The owner's exact recipe, when ready to run it against the shared cluster:**
+```bash
+# 1. Confirm the target index actually needs it (optional sanity check):
+curl -s -X GET "$OPENSEARCH_STORAGE_ENDPOINT/evals_runs/_mapping" \
+  -u "$OPENSEARCH_STORAGE_USERNAME:$OPENSEARCH_STORAGE_PASSWORD" \
+  | jq '.evals_runs.mappings.properties.metrics'
+# If this prints a `dynamic` key, it's already fixed. If it prints only
+# `properties` with more than the 4 legacy metric names, it's poisoned.
+# 2. Run the reindex via the running server's admin API (recreates the
+#    index from the current INDEX_MAPPINGS and copies every document
+#    across; the same doc-count-validated, recovery-safe path
+#    reindexSingleIndex() has always used for keyword-type mismatch fixes):
+curl -s -X POST "http://localhost:4001/api/storage/reindex" \
+  -H 'Content-Type: application/json' \
+  -d '{"index": "evals_runs"}'
+```
+Caveats to read before running this against the shared cluster:
+- **No write lock during a manual `/api/storage/reindex` call.** The
+  auto-fix boot path (`fixIndexMappings()`) acquires a process-local
+  migration lock around the reindex; the manual admin route calls
+  `reindexSingleIndex()` directly and does **not** (pre-existing gap in
+  `server/routes/storage/admin.ts`, not introduced by this PR — flagged here,
+  not fixed, since it's out of scope for this change). Run it during a quiet
+  window (no in-flight evaluation runs writing reports) to avoid a write
+  racing the index delete/recreate step.
+- It touches only `evals_runs`. The already-known-poisoned `evals_experiments`
+  (800+ stale `results.*` fields, per the incident notes) uses the identical
+  recipe (`{"index": "evals_experiments"}`) — that cleanup is separately
+  planned by ops; this PR does not touch or schedule it.
+- Document count is validated before the temporary index is deleted; if the
+  copy-back count doesn't match, the error message names the surviving temp
+  index (`evals_runs_reindex_temp`) for manual recovery — nothing is deleted
+  until the counts are confirmed equal.
+## Tests
+- Unit (`tests/unit/server/constants/indexMappings.test.ts`): mapping-shape
+  assertions — `dynamic: false` + typed legacy properties on both `metrics`
+  and `matcherResults.judgeMetrics`; pre-existing `enabled:false` fields stay
+  disabled; every field `OpenSearchRunOperations.search()` queries stays
+  explicitly mapped.
+- Integration, real OpenSearch
+  (`tests/integration/services/storage/testCaseRunMetricsMappingGrowth.integration.test.ts`):
+  writes one report with 1000+ distinct custom `metrics`/`judgeMetrics` names
+  (500 report-level + 125 `judge()` calls × 4 dimensions), asserts it
+  round-trips correctly and the index's total mapped-field count does not
+  grow; asserts the query-audit fields stay queryable.
+- Integration, real OpenSearch, migration recipe
+  (`tests/integration/services/storage/evalsRunsMappingMigrationRecipe.integration.test.ts`):
+  deliberately poisons a throwaway index the old way, runs the *existing*
+  `reindexSingleIndex()`, asserts the mapping resets to `dynamic: false` and
+  all document data survives byte-for-byte.
+Both integration suites skip gracefully (with a console warning) if no
+OpenSearch cluster is reachable at `TEST_OPENSEARCH_ENDPOINT` (default
+`http://localhost:9200`) — the unit suite covers the mapping-shape assertions
+unconditionally.

package/docs/skills/AGENT_HEALTH.md CHANGED Viewed

@@ -167,7 +167,7 @@ run a JSON file:
 ```bash
 # `-f` accepts BOTH JSON test-case files and code SDK (.eval.js / .eval.ts) files
-npx @opensearch-project/agent-health benchmark -f ./evals/demo.eval.js -a my-agent
+npx @opensearch-project/agent-health benchmark -f ./examples/eval-files/demo.eval.js -a my-agent
 ```
 They produce **per-matcher results** (`matcherResults[]`) instead of a single
@@ -303,6 +303,60 @@ Repeat until all high-priority issues are resolved.
 | Server / config issues | `npx agent-health doctor` (checks config + connectivity) |
 | "OpenSearch storage not configured" | Fine for local use — file-based storage is the default. Set `OPENSEARCH_STORAGE_*` only for shared / production persistence. |
+> **Note (benchmark execution vs. reads):** file-based storage covers reads,
+> single `run`s, and sample data, but **executing a multi-case `benchmark`
+> currently requires an OpenSearch storage client** — without one the execute
+> path returns `Cannot execute in sample-only mode`. Point `OPENSEARCH_STORAGE_*`
+> at a cluster (a local security-disabled Docker OpenSearch on plain HTTP with
+> `authType=none` is enough) before running `benchmark`.
+---
+## Benchmarking a local (subprocess) agent — gotchas
+When you wrap a local CLI agent (Kiro, Claude Code, Pi, or your own script) as a
+**subprocess** agent and run a `benchmark`, these are the traps that bite first
+— check them before blaming the agent:
+1. **Pin a real judge model — don't leave it on `demo`.** The `demo` provider is
+   a mock judge that returns high pass rates without calling an LLM, so a run
+   can look like "100% pass" while nothing was actually judged. Set the run's
+   judge model to a real Bedrock model, e.g.
+   `us.anthropic.claude-sonnet-4-5-20250929-v1:0`, and confirm it's invocable in
+   your account (`aws bedrock ... ` / `doctor`). If your pass rate looks too
+   good, check the judge provider first.
+2. **Raise the subprocess timeout for slow agents.** The subprocess connector
+   defaults to a 5-minute (`300000` ms) timeout. A real ops/RCA agent can run
+   ~10 min. Set it in your agent's `connectorConfig`:
+   ```ts
+   { key: 'my-ops-agent', connectorType: 'subprocess',
+     connectorConfig: { timeout: 1200000 /* 20 min */, /* ... */ } }
+   ```
+3. **Fail loud on a wrong agent name.** If your wrapper points at an agent key
+   that doesn't exist, the underlying CLI may silently fall back to a default
+   agent — so you benchmark the wrong thing. Echo the resolved agent name in
+   your wrapper and eyeball the first trajectory.
+4. **Long runs + CLI SSE disconnects.** On runs longer than a few minutes the
+   CLI's streaming connection can drop and report `0/0` / `fetch failed` **while
+   the server keeps going**. The results are still persisted — read them back
+   from storage (`list runs` / the UI run inspector / `--export`) rather than
+   trusting the CLI summary.
+5. **Watch for a stale server on the port.** If you patch/upgrade the package
+   but a previously-started `npx` server is still bound to port 4001, your runs
+   are served by the old code. Confirm which process owns the port
+   (`lsof -i :4001` / a `/proc` sweep) and restart the one you actually patched
+   (the server loads `server/dist/app.js`, not `index.js`).
+6. **Judge `CredentialsProviderError` after a session rotates.** If your AWS
+   sandbox session rotates, `AWS_SHARED_CREDENTIALS_FILE` can point at a stale/
+   empty file and the judge fails. Re-vend credentials and restart the server
+   with the AWS env explicit (`aws sts get-caller-identity` to confirm first).
+7. **Local OpenSearch dying (exit 255) = memory pressure.** Give the container
+   enough heap and run it with `--restart=unless-stopped`.
+8. **Case schema is strict.** Test cases need `expectedOutcomes` (a
+   **string[]**, not a singular `expectedOutcome` string) and a capitalized
+   `difficulty` (`Easy` | `Medium` | `Hard`). Validate your converter output
+   against a known-good case before importing 16 of them.
 ---
 ## Server API Reference

package/examples/eval-files/demo.eval.js CHANGED Viewed

@@ -18,7 +18,7 @@
  *     -H 'Content-Type: application/json' \
  *     -d '{
  *       "name":"SDK Demo",
- *       "sources":[{"type":"code-import","filenames":["evals/demo.eval.js"],"testCaseIds":[]}],
+ *       "sources":[{"type":"code-import","filenames":["examples/eval-files/demo.eval.js"],"testCaseIds":[]}],
  *       "agentKey":"observio",
  *       "modelId":"claude-sonnet"
  *     }'

package/examples/eval-files/ops-rca-classification.eval.js ADDED Viewed

@@ -0,0 +1,71 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * Worked example — evaluating an ops / RCA agent that triages a ticket.
+ *
+ * The agent under test reads a ticket and emits a structured classification:
+ *   - ticketType:  "latency" | "fault"        (deterministic — exact match)
+ *   - rootCause:   one of a fixed category set (deterministic — exact match)
+ *   - sop:         a recommended runbook        (non-deterministic — LLM judge)
+ *
+ * This shows the two check styles side by side:
+ *   • deterministic  → expect(...).to.equal(...) on the parsed output
+ *   • non-deterministic → judge(result, '<natural-language claim>')
+ *
+ * WEIGHTED SCORING (root-cause 60%, SOP, metrics, latency 10%):
+ * per-matcher pass/fail lives here, but the *single weighted aggregate score*
+ * across criteria is defined once in a custom EVALUATOR, not in the test body.
+ * Attach it at run time with `evaluatorId` — see ops-rca-evaluator.json next to
+ * this file. Run:
+ *
+ *   npx @opensearch-project/agent-health benchmark \
+ *     -f ./examples/eval-files/ops-rca-classification.eval.js -a my-ops-agent
+ *
+ * Docs: ../../docs/SDK.md   Instrumentation: ../../docs/INSTRUMENT_WITH_OTEL.md
+ */
+const { test, expect } = require('@opensearch-project/agent-health');
+// The agent's allowed root-cause categories — deterministic ground truth.
+const ROOT_CAUSE_CATEGORIES = [
+  'dependency_outage',
+  'resource_exhaustion',
+  'config_error',
+  'code_regression',
+  'network',
+];
+test('ticket-123-db-outage', {
+  prompt: 'Triage ticket TICKET-123 and return your classification table.',
+  description: 'DB dependency outage — must classify as fault + dependency_outage',
+  context: [
+    {
+      description: 'Ticket body',
+      value:
+        'TICKET-123: payment-service returning 500s since 10:30. ' +
+        'Logs: "Connection refused to database-primary:5432". p99 latency normal until errors began.',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Medium', 'agent:ops', 'type:fault'],
+}, async function ({ agent, judge }) {
+  const result = await agent.run();
+  // ── Deterministic checks — exact classification, no LLM, $0 ──────────────
+  const out = result.parsedOutput() || {}; // agent emits JSON classification
+  expect(out.ticketType).to.equal('fault');
+  expect(ROOT_CAUSE_CATEGORIES).to.include(out.rootCause);
+  expect(out.rootCause).to.equal('dependency_outage');
+  // Prove it actually investigated rather than guessing.
+  expect(result.trajectory).to.haveStepsOfType('action');
+  // ── Non-deterministic checks — LLM judge on the free-text SOP ────────────
+  await judge(result, 'Recommends a runbook appropriate for a database dependency outage');
+  await judge(result, 'Explains that payment-service cannot reach database-primary as the root cause');
+  // ── Budget guard (feeds the "latency 10%" weight via the evaluator) ──────
+  expect(result).to.haveCompletedWithin(120_000);
+});

package/examples/eval-files/ops-rca-evaluator.json ADDED Viewed

@@ -0,0 +1,15 @@
+{
+  "name": "Ops RCA (weighted)",
+  "description": "Weighted scoring for a ticket-triage / RCA ops agent. Root-cause accuracy dominates; latency is a small tie-breaker. Attach to a run via evaluatorId.",
+  "systemPrompt": "You are evaluating an ops agent that triages a support ticket. The agent must (1) classify the ticket type (latency vs fault), (2) identify the root cause from a fixed category set, (3) recommend an appropriate SOP/runbook, and (4) check the relevant metrics. Score each metric from 0 to 100.\n\nCRITICAL CRITERIA:\n- root_cause_accuracy is the PRIMARY metric. A wrong root cause is a failed triage regardless of everything else.\n- sop_selection: did the agent recommend a runbook appropriate for the identified root cause?\n- relevant_metrics: did the agent inspect the metrics/logs that actually matter for this failure mode?\n- latency: score higher when the agent reaches a correct answer in fewer steps / less wall-clock time.\n\nReturn pass_fail_status, reasoning, a metrics object with the four numeric scores, and improvement_strategies.",
+  "scoringConfig": {
+    "metrics": [
+      { "name": "root_cause_accuracy", "weight": 0.6, "scale": 100 },
+      { "name": "sop_selection", "weight": 0.2, "scale": 100 },
+      { "name": "relevant_metrics", "weight": 0.1, "scale": 100 },
+      { "name": "latency", "weight": 0.1, "scale": 100 }
+    ],
+    "passThreshold": 80,
+    "scale": 100
+  }
+}

package/examples/eval-files/sdk-demo.eval.js ADDED Viewed

@@ -0,0 +1,72 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * SDK demo eval — three test cases that show the spectrum of evaluation
+ * methods the code-based SDK supports:
+ *
+ *   1. mock-says-hello       (deterministic)  — agent invoked, only chai matchers
+ *   2. mock-rca-judged       (agentic)        — agent invoked + LLM judge matcher
+ *   3. data-only-no-prompt   (deterministic)  — no agent call at all
+ *
+ * Run with:
+ *   AH_PORT=4002 npx @opensearch-project/agent-health benchmark \
+ *     -f examples/eval-files/sdk-demo.eval.js -a demo
+ */
+const { test, expect } = require('@opensearch-project/agent-health');
+// ─────────────────────────────────────────────────────────────────────────────
+// 1. Deterministic — agent runs, all assertions are local chai matchers
+// ─────────────────────────────────────────────────────────────────────────────
+test('mock-says-hello', {
+  prompt: 'Say hello in one short sentence.',
+  description: 'Mock agent must produce a non-empty response within 30s',
+  labels: ['category:Smoke', 'difficulty:Easy', 'method:deterministic'],
+}, async function ({ agent }) {
+  const result = await agent.run();
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(30_000);
+});
+// ─────────────────────────────────────────────────────────────────────────────
+// 2. Agentic (hybrid) — deterministic preflight + LLM judge for semantic claim
+// ─────────────────────────────────────────────────────────────────────────────
+test('mock-rca-judged', {
+  prompt: 'Diagnose why the payment service is failing and explain the root cause.',
+  description: 'Hybrid: structural checks first, then LLM judge for semantic correctness',
+  context: [
+    {
+      description: 'Error log',
+      value: 'ERROR 2026-05-20 10:31:22 [payment-service] Connection refused to db-primary:5432',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Medium', 'method:agentic'],
+}, async function ({ agent, judge }) {
+  const result = await agent.run();
+  // Cheap deterministic preflight — fail fast before spending $ on the judge
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(60_000);
+  // LLM judge — produces a structured matcher verdict with score + reasoning
+  await judge(result, 'Mentions the payment service or its database connection failure');
+});
+// ─────────────────────────────────────────────────────────────────────────────
+// 3. Deterministic, no prompt — agent never invoked, $0 / 0ms agent step
+// ─────────────────────────────────────────────────────────────────────────────
+test('data-only-no-prompt', {
+  description: 'Pure data check; agent invocation skipped entirely',
+  labels: ['category:Data Quality', 'difficulty:Easy', 'method:deterministic'],
+}, function ({ result }) {
+  expect(result.durationMs).to.equal(0);
+  expect(result.trajectory).to.have.length(0);
+  expect(2 + 2).to.equal(4);
+});