@opensearch-project/agent-health 0.4.0 → 0.6.0

package/examples/config/agent-health.config.example.ts

@@ -0,0 +1,155 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Agent Health Configuration
+ *
+ * Copy this file to `agent-health.config.ts` and customize for your environment.
+ * The config file is automatically detected in the current working directory.
+ *
+ * By default, user-defined agents and models are merged with the built-in defaults.
+ * Set `extends: false` to use only your agents and models.
+ *
+ * Supported config file names (in priority order):
+ *   - agent-health.config.ts
+ *   - agent-health.config.js
+ *   - agent-health.config.mjs
+ */
+
+export default {
+  // Custom agents are merged with built-in agents by default.
+  // Each agent `key` must be unique — if it matches a built-in key,
+  // it will override that built-in agent.
+  agents: [
+    // Example 1: REST connector (synchronous JSON response)
+    {
+      key: "my-rest-agent",
+      name: "My REST Agent",
+      endpoint: "http://localhost:8000/api/agent",
+      connectorType: "rest",
+      useTraces: true,           // Enable OpenTelemetry trace collection
+    },
+
+    // Example 2: Streaming connector (Server-Sent Events)
+    {
+      key: "my-streaming-agent",
+      name: "My Streaming Agent",
+      endpoint: "http://localhost:9000/agent/stream",
+      connectorType: "agui-streaming",
+      useTraces: false,
+    },
+
+    // Example 3: CLI tool connector
+    // {
+    //   key: "my-cli-agent",
+    //   name: "My CLI Agent",
+    //   endpoint: "/usr/local/bin/my-agent",
+    //   connectorType: "subprocess",
+    //   useTraces: false,
+    // },
+
+    // Example 4: LangGraph REST connector (non-streaming invoke)
+    // {
+    //   key: "my-langgraph-agent",
+    //   name: "My LangGraph Agent",
+    //   endpoint: "http://localhost:8123",
+    //   connectorType: "langgraph",
+    //   connectorConfig: {
+    //     graphId: "agent",        // Assistant/graph ID (defaults to "agent")
+    //     // threadId: "...",      // Optional thread ID for multi-turn
+    //   },
+    //   useTraces: true,
+    // },
+
+    // Example 5: Amazon Strands connector (Bedrock Agent Runtime)
+    // {
+    //   key: "my-strands-agent",
+    //   name: "My Strands Agent",
+    //   endpoint: "ABCDEF1234",     // Bedrock Agent ID
+    //   connectorType: "strands",
+    //   connectorConfig: {
+    //     agentAliasId: "TSTALIASID",  // Agent alias ID
+    //     region: "us-west-2",          // AWS region (defaults to AWS_REGION)
+    //   },
+    //   useTraces: false,
+    // },
+
+    // Example 6: Agent with authentication hook
+    // {
+    //   key: "authenticated-agent",
+    //   name: "Authenticated Agent",
+    //   endpoint: "https://api.example.com/agent",
+    //   connectorType: "rest",
+    //   useTraces: true,
+    //   hooks: {
+    //     beforeRequest: async ({ endpoint, payload, headers }) => {
+    //       return {
+    //         endpoint,
+    //         payload,
+    //         headers: {
+    //           ...headers,
+    //           'Authorization': `Bearer ${process.env.API_TOKEN}`,
+    //         },
+    //       };
+    //     },
+    //   },
+    // },
+
+    // Example 7: Claude Code with MCP servers (competitive evaluation)
+    // Uses a standard MCP config JSON file (same format as ~/.claude.json mcpServers)
+    // {
+    //   key: "claude-code-eval",
+    //   name: "Claude Code (Eval)",
+    //   endpoint: "claude",
+    //   connectorType: "claude-code",
+    //   useTraces: true,
+    //   connectorConfig: {
+    //     dangerouslySkipPermissions: true,
+    //     appendSystemPrompt: "You are an observability agent...",
+    //     allowedTools: ["mcp__aws-prometheus__*", "mcp__aws-cloudwatch__*", "Bash"],
+    //     mcpConfigPath: "./mcp-config.json",  // standard MCP config file
+    //     strictMcpConfig: true,
+    //     env: { CLAUDE_CODE_USE_BEDROCK: "1", AWS_PROFILE: "Bedrock" },
+    //   },
+    // },
+  ],
+
+  // --- Data sources (OpenSearch clusters) ----------------------------------
+  // storage:       where eval results are persisted (omit for file-based).
+  // observability: where agent traces/logs are read from (Traces tab).
+  // Precedence: agent-health.config.json (Settings UI) -> these TS fields ->
+  //   OPENSEARCH_STORAGE_*/OPENSEARCH_LOGS_* env -> file-based fallback.
+  // Read secrets from process.env so this committed file carries none.
+  //
+  // storage: {
+  //   endpoint: process.env.OPENSEARCH_STORAGE_ENDPOINT,
+  //   authType: 'sigv4',          // 'none' | 'basic' | 'sigv4'
+  //   awsRegion: 'us-east-1',
+  //   awsService: 'es',           // 'es' (managed) | 'aoss' (serverless)
+  //   awsProfile: process.env.AWS_PROFILE,
+  // },
+  //
+  // observability: {
+  //   endpoint: process.env.OPENSEARCH_LOGS_ENDPOINT,
+  //   authType: 'sigv4',
+  //   awsRegion: 'us-east-1',
+  //   indexes: { traces: 'otel-v1-apm-span-*', logs: 'ml-commons-logs-*' },
+  // },
+
+  // Custom models (merged with built-in models by default)
+  // models: [
+  //   {
+  //     key: "my-model",
+  //     model_id: "us.anthropic.claude-sonnet-4-20250514-v1:0",
+  //     display_name: "My Model",
+  //     provider: "bedrock",       // "bedrock" | "openai-compatible" | "demo"
+  //     context_window: 200000,
+  //     max_output_tokens: 4096,
+  //   },
+  // ],
+
+  // Set to false to ignore all built-in agents and models
+  // extends: false,
+};

package/examples/connectors/echo-connector.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Example: a custom connector that wraps a simple HTTP echo agent.
+ *
+ * This is the smallest realistic shape of a connector. It implements the
+ * three abstract members of `BaseConnector` (`buildPayload`, `execute`,
+ * `parseResponse`) plus the three readonly identity fields (`type`,
+ * `name`, `supportsStreaming`).
+ *
+ * Drop this file at `connectors/echo.ts` in your project and register it
+ * from your `agent-health.config.ts`:
+ *
+ * @example
+ * ```ts
+ * import { defineConfig, registerConnector } from '@opensearch-project/agent-health';
+ * import { EchoConnector } from './connectors/echo';
+ *
+ * registerConnector(new EchoConnector());
+ *
+ * export default defineConfig({
+ *   agents: [
+ *     {
+ *       key: 'echo',
+ *       name: 'Echo Agent',
+ *       endpoint: 'http://localhost:8080/echo',
+ *       connectorType: 'rest',  // or any registered protocol
+ *     },
+ *   ],
+ * });
+ * ```
+ *
+ * To understand the lifecycle this connector plugs into — abort signal
+ * handling, progress callbacks, trajectory shape, raw-event replay —
+ * read the base class and types directly in the installed package:
+ *
+ *     node_modules/@opensearch-project/agent-health/lib/dist/services/connectors/base/BaseConnector.js
+ *     node_modules/@opensearch-project/agent-health/lib/dist/services/connectors/types.d.ts
+ *
+ * The compiled `.js` preserves the original JSDoc, and the matching
+ * `.d.ts` next to it gives you the precise type contract — no need to
+ * cross-reference an arbitrary GitHub commit, you're reading exactly
+ * the version your `package.json` resolved to.
+ *
+ * NOTE: `ConnectorProtocol` is a closed string-literal union of the
+ * built-in protocol ids ('rest', 'agui-streaming', 'subprocess', ...).
+ * The simplest path for a custom connector is to reuse one of those ids
+ * (e.g. 'rest' for HTTP-JSON agents) and replace the built-in connector
+ * via `registerConnector(new EchoConnector())`. If you need a brand-new
+ * protocol id, fork this repo and widen the `ConnectorProtocol` union
+ * in `services/connectors/types.ts`.
+ */
+
+import {
+  BaseConnector,
+  type ConnectorAuth,
+  type ConnectorRequest,
+  type ConnectorResponse,
+  type ConnectorProgressCallback,
+  type ConnectorRawEventCallback,
+} from '@opensearch-project/agent-health';
+
+import type { TrajectoryStep } from '@opensearch-project/agent-health';
+
+export class EchoConnector extends BaseConnector {
+  // Replaces the built-in 'rest' connector when registered. Pick a
+  // different id (and widen ConnectorProtocol in your fork) if you
+  // want to coexist with the default REST connector.
+  readonly type = 'rest' as const;
+  readonly name = 'Echo Agent';
+  readonly supportsStreaming = false;
+
+  /**
+   * Build the JSON payload sent to the agent's HTTP endpoint.
+   * Called by `execute()` and also when re-processing stored raw events
+   * during trace replay (see `parseResponse`).
+   */
+  buildPayload(request: ConnectorRequest): { message: string } {
+    return { message: request.testCase.initialPrompt };
+  }
+
+  /**
+   * Single-turn HTTP call → single response.
+   */
+  async execute(
+    endpoint: string,
+    request: ConnectorRequest,
+    _auth: ConnectorAuth,
+    onProgress?: ConnectorProgressCallback,
+    _onRawEvent?: ConnectorRawEventCallback,
+  ): Promise<ConnectorResponse> {
+    const startedAt = Date.now();
+
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(this.buildPayload(request)),
+    });
+    if (!res.ok) {
+      throw new Error(`Echo agent failed: ${res.status} ${res.statusText}`);
+    }
+    const json = (await res.json()) as { reply: string };
+
+    // Build a single-step trajectory and emit progress for live UIs.
+    const step = this.createStep('response', json.reply);
+    onProgress?.(step);
+
+    return {
+      success: true,
+      response: json.reply,
+      trajectory: [step],
+      rawEvents: [{ type: 'ECHO_REPLY', reply: json.reply }],
+      durationMs: Date.now() - startedAt,
+      runId: `echo-${startedAt}`,
+    };
+  }
+
+  /**
+   * Re-derive trajectory from previously stored raw events. Used by
+   * the run replayer / trace recovery path.
+   */
+  parseResponse(rawResponse: unknown): TrajectoryStep[] {
+    const events = Array.isArray(rawResponse) ? rawResponse : [];
+    return events
+      .filter((e: { type?: string }) => e?.type === 'ECHO_REPLY')
+      .map((e: { reply: string }) => this.createStep('response', e.reply));
+  }
+}

package/examples/eval-files/demo.eval.js

@@ -0,0 +1,128 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Demo eval — five test cases that show off the SDK end-to-end against
+ * the live Observio agent.
+ *
+ *   1. observio-says-hello          Pure deterministic, no LLM (free, fast)
+ *   2. observio-uses-a-tool         Trajectory inspection — must invoke at least one tool
+ *   3. observio-rca-is-coherent     Hybrid — deterministic + LLM judge
+ *   4. labels-only-no-prompt        No agent invocation at all (data-only test)
+ *   5. budget-aware                 Uses traces fixture for token-budget assertion
+ *
+ * Run with:
+ *   curl -sN -X POST http://localhost:4002/api/storage/evaluation-runs \
+ *     -H 'Content-Type: application/json' \
+ *     -d '{
+ *       "name":"SDK Demo",
+ *       "sources":[{"type":"code-import","filenames":["examples/eval-files/demo.eval.js"],"testCaseIds":[]}],
+ *       "agentKey":"observio",
+ *       "modelId":"claude-sonnet"
+ *     }'
+ */
+
+const { test, expect } = require('@opensearch-project/agent-health');
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 1: Pure deterministic — no LLM, $0, fully reproducible
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-says-hello', {
+  prompt: 'Say hello to me in one short sentence.',
+  description: 'Smoke test: agent produces a non-trivial response',
+  labels: ['category:Smoke', 'difficulty:Easy', 'demo', 'agent:observio'],
+}, function ({ result }) {
+  // Trajectory must exist
+  expect(result.trajectory).to.have.length.greaterThan(0);
+
+  // Final output is non-empty
+  expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+
+  // Reasonable timing budget
+  expect(result).to.haveCompletedWithin(60_000);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 2: Trajectory inspection — proves the agent actually investigates
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-uses-a-tool', {
+  prompt: 'Find the source of the most recent error log entry.',
+  description: 'Agent must call at least one tool, not just answer from memory',
+  labels: ['category:Tool Use', 'difficulty:Medium', 'demo', 'agent:observio'],
+}, function ({ result }) {
+  // At least one action step (tool invocation)
+  expect(result.trajectory).to.haveStepsOfType('action');
+
+  // The trajectory accessor sugar makes the assertion read like prose
+  const tools = result.trajectory.toolCalls();
+  expect(tools).to.have.length.greaterThan(0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 3: Hybrid — deterministic preflight + targeted LLM judge
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-rca-is-coherent', {
+  prompt: 'Diagnose why the payment service is failing and explain the root cause.',
+  description: 'Hybrid: cheap structural checks first, then LLM judge for semantic correctness',
+  context: [
+    {
+      description: 'Error log',
+      value: 'ERROR 2024-01-15 10:31:22 [payment-service] Connection refused to database-primary:5432',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Hard', 'demo', 'agent:observio', 'hybrid'],
+}, async function ({ result, judge }) {
+  // Cheap deterministic preflight — fail fast, never spend $ on the judge
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(120_000);
+  expect(result.agentOutput).to.haveOutputMatching(/payment[- ]service/i);
+
+  // Now invoke the LLM judge for the semantic claim. This produces a
+  // 'hybrid' evaluation method on the resulting report.
+  await judge(result, 'Correctly identifies that the payment-service cannot connect to its database');
+  await judge(result, 'Provides a plausible root-cause hypothesis (e.g. database down, network issue)');
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 4: No prompt at all — pure data-only test, agent never invoked
+// ─────────────────────────────────────────────────────────────────────────────
+//
+// Useful for tests that assert against external state, fixtures, or static
+// data without needing any agent involvement.
+
+test('labels-only-no-prompt', {
+  description: 'Verify a fixture file matches a baseline (no agent involvement)',
+  labels: ['category:Data Quality', 'difficulty:Easy', 'demo'],
+}, function ({ result }) {
+  // No agent ran — durationMs is 0, trajectory empty
+  expect(result.durationMs).to.equal(0);
+  expect(result.trajectory).to.have.length(0);
+
+  // The actual deterministic check (could load a JSON file, compute, etc.)
+  const baseline = { name: 'demo', version: 1 };
+  expect(baseline.version).to.equal(1);
+  expect(baseline.name).to.equal('demo');
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 5: Budget-aware — uses the traces fixture for token / span timing checks
+// ─────────────────────────────────────────────────────────────────────────────
+//
+// `traces` is pre-loaded by the runner when the agent emits OTel spans. When
+// no traces are available the accessor returns 0 / empty for all helpers,
+// so this test still passes (vacuously) on agents without instrumentation.
+
+test('budget-aware', {
+  prompt: 'Summarize the system in one paragraph.',
+  description: 'Token budget + span-timing assertions via the traces fixture',
+  labels: ['category:Budget', 'difficulty:Easy', 'demo', 'agent:observio'],
+}, function ({ result, traces }) {
+  expect(result.agentOutput.length).to.be.greaterThan(0);
+  // Cheap budget cap — fails if the agent burned more than 50k tokens
+  expect(traces.totalTokens).to.be.lessThan(50_000);
+});

package/examples/eval-files/ops-rca-classification.eval.js

@@ -0,0 +1,71 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Worked example — evaluating an ops / RCA agent that triages a ticket.
+ *
+ * The agent under test reads a ticket and emits a structured classification:
+ *   - ticketType:  "latency" | "fault"        (deterministic — exact match)
+ *   - rootCause:   one of a fixed category set (deterministic — exact match)
+ *   - sop:         a recommended runbook        (non-deterministic — LLM judge)
+ *
+ * This shows the two check styles side by side:
+ *   • deterministic  → expect(...).to.equal(...) on the parsed output
+ *   • non-deterministic → judge(result, '<natural-language claim>')
+ *
+ * WEIGHTED SCORING (root-cause 60%, SOP, metrics, latency 10%):
+ * per-matcher pass/fail lives here, but the *single weighted aggregate score*
+ * across criteria is defined once in a custom EVALUATOR, not in the test body.
+ * Attach it at run time with `evaluatorId` — see ops-rca-evaluator.json next to
+ * this file. Run:
+ *
+ *   npx @opensearch-project/agent-health benchmark \
+ *     -f ./examples/eval-files/ops-rca-classification.eval.js -a my-ops-agent
+ *
+ * Docs: ../../docs/SDK.md   Instrumentation: ../../docs/INSTRUMENT_WITH_OTEL.md
+ */
+
+const { test, expect } = require('@opensearch-project/agent-health');
+
+// The agent's allowed root-cause categories — deterministic ground truth.
+const ROOT_CAUSE_CATEGORIES = [
+  'dependency_outage',
+  'resource_exhaustion',
+  'config_error',
+  'code_regression',
+  'network',
+];
+
+test('ticket-123-db-outage', {
+  prompt: 'Triage ticket TICKET-123 and return your classification table.',
+  description: 'DB dependency outage — must classify as fault + dependency_outage',
+  context: [
+    {
+      description: 'Ticket body',
+      value:
+        'TICKET-123: payment-service returning 500s since 10:30. ' +
+        'Logs: "Connection refused to database-primary:5432". p99 latency normal until errors began.',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Medium', 'agent:ops', 'type:fault'],
+}, async function ({ agent, judge }) {
+  const result = await agent.run();
+
+  // ── Deterministic checks — exact classification, no LLM, $0 ──────────────
+  const out = result.parsedOutput() || {}; // agent emits JSON classification
+  expect(out.ticketType).to.equal('fault');
+  expect(ROOT_CAUSE_CATEGORIES).to.include(out.rootCause);
+  expect(out.rootCause).to.equal('dependency_outage');
+
+  // Prove it actually investigated rather than guessing.
+  expect(result.trajectory).to.haveStepsOfType('action');
+
+  // ── Non-deterministic checks — LLM judge on the free-text SOP ────────────
+  await judge(result, 'Recommends a runbook appropriate for a database dependency outage');
+  await judge(result, 'Explains that payment-service cannot reach database-primary as the root cause');
+
+  // ── Budget guard (feeds the "latency 10%" weight via the evaluator) ──────
+  expect(result).to.haveCompletedWithin(120_000);
+});

package/examples/eval-files/ops-rca-evaluator.json

@@ -0,0 +1,15 @@
+{
+  "name": "Ops RCA (weighted)",
+  "description": "Weighted scoring for a ticket-triage / RCA ops agent. Root-cause accuracy dominates; latency is a small tie-breaker. Attach to a run via evaluatorId.",
+  "systemPrompt": "You are evaluating an ops agent that triages a support ticket. The agent must (1) classify the ticket type (latency vs fault), (2) identify the root cause from a fixed category set, (3) recommend an appropriate SOP/runbook, and (4) check the relevant metrics. Score each metric from 0 to 100.\n\nCRITICAL CRITERIA:\n- root_cause_accuracy is the PRIMARY metric. A wrong root cause is a failed triage regardless of everything else.\n- sop_selection: did the agent recommend a runbook appropriate for the identified root cause?\n- relevant_metrics: did the agent inspect the metrics/logs that actually matter for this failure mode?\n- latency: score higher when the agent reaches a correct answer in fewer steps / less wall-clock time.\n\nReturn pass_fail_status, reasoning, a metrics object with the four numeric scores, and improvement_strategies.",
+  "scoringConfig": {
+    "metrics": [
+      { "name": "root_cause_accuracy", "weight": 0.6, "scale": 100 },
+      { "name": "sop_selection", "weight": 0.2, "scale": 100 },
+      { "name": "relevant_metrics", "weight": 0.1, "scale": 100 },
+      { "name": "latency", "weight": 0.1, "scale": 100 }
+    ],
+    "passThreshold": 80,
+    "scale": 100
+  }
+}

package/examples/eval-files/sdk-demo.eval.js

@@ -0,0 +1,72 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * SDK demo eval — three test cases that show the spectrum of evaluation
+ * methods the code-based SDK supports:
+ *
+ *   1. mock-says-hello       (deterministic)  — agent invoked, only chai matchers
+ *   2. mock-rca-judged       (agentic)        — agent invoked + LLM judge matcher
+ *   3. data-only-no-prompt   (deterministic)  — no agent call at all
+ *
+ * Run with:
+ *   AH_PORT=4002 npx @opensearch-project/agent-health benchmark \
+ *     -f examples/eval-files/sdk-demo.eval.js -a demo
+ */
+
+const { test, expect } = require('@opensearch-project/agent-health');
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 1. Deterministic — agent runs, all assertions are local chai matchers
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('mock-says-hello', {
+  prompt: 'Say hello in one short sentence.',
+  description: 'Mock agent must produce a non-empty response within 30s',
+  labels: ['category:Smoke', 'difficulty:Easy', 'method:deterministic'],
+}, async function ({ agent }) {
+  const result = await agent.run();
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(30_000);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 2. Agentic (hybrid) — deterministic preflight + LLM judge for semantic claim
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('mock-rca-judged', {
+  prompt: 'Diagnose why the payment service is failing and explain the root cause.',
+  description: 'Hybrid: structural checks first, then LLM judge for semantic correctness',
+  context: [
+    {
+      description: 'Error log',
+      value: 'ERROR 2026-05-20 10:31:22 [payment-service] Connection refused to db-primary:5432',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Medium', 'method:agentic'],
+}, async function ({ agent, judge }) {
+  const result = await agent.run();
+
+  // Cheap deterministic preflight — fail fast before spending $ on the judge
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(60_000);
+
+  // LLM judge — produces a structured matcher verdict with score + reasoning
+  await judge(result, 'Mentions the payment service or its database connection failure');
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 3. Deterministic, no prompt — agent never invoked, $0 / 0ms agent step
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('data-only-no-prompt', {
+  description: 'Pure data check; agent invocation skipped entirely',
+  labels: ['category:Data Quality', 'difficulty:Easy', 'method:deterministic'],
+}, function ({ result }) {
+  expect(result.durationMs).to.equal(0);
+  expect(result.trajectory).to.have.length(0);
+  expect(2 + 2).to.equal(4);
+});

package/examples/eval-files/sdk-describe-demo.eval.js

@@ -0,0 +1,50 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * SDK demo with describe() grouping. Each describe() block declares a
+ * benchmark; CLI auto-creates one benchmark per group and runs them.
+ */
+
+const { describe, test, expect } = require('@opensearch-project/agent-health');
+
+describe('RCA Suite', () => {
+  test('payment service is the root cause', {
+    prompt: 'Diagnose why the payment service is failing.',
+    context: [{ description: 'Error log', value: 'ERROR [payment-service] Connection refused to db-primary:5432' }],
+    labels: ['category:RCA', 'difficulty:Medium'],
+  }, async function ({ agent, judge }) {
+    const result = await agent.run();
+    expect(result.trajectory).to.have.length.greaterThan(0);
+    expect(result).to.haveCompletedWithin(60_000);
+    await judge(result, 'Mentions the payment service or its database connection failure');
+  });
+
+  test('budget cap', {
+    prompt: 'Summarize the system in one paragraph.',
+    labels: ['category:Budget', 'difficulty:Easy'],
+  }, async function ({ agent, traces }) {
+    const result = await agent.run();
+    expect(result.agentOutput.length).to.be.greaterThan(0);
+    expect(traces.totalTokens).to.be.lessThan(50_000);
+  });
+});
+
+describe('Smoke', () => {
+  test('says hello', {
+    prompt: 'Say hello in one short sentence.',
+    labels: ['category:Smoke', 'difficulty:Easy'],
+  }, async function ({ agent }) {
+    const result = await agent.run();
+    expect(result.trajectory).to.have.length.greaterThan(0);
+    expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+    expect(result).to.haveCompletedWithin(30_000);
+  });
+});
+
+// Tests outside any describe() — go to the file-default benchmark
+test('orphan-deterministic', function () {
+  expect(2 + 2).to.equal(4);
+});