@opensearch-project/agent-health 0.3.0 → 0.5.2

package/examples/config/agent-health.config.example.ts

@@ -0,0 +1,155 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Agent Health Configuration
+ *
+ * Copy this file to `agent-health.config.ts` and customize for your environment.
+ * The config file is automatically detected in the current working directory.
+ *
+ * By default, user-defined agents and models are merged with the built-in defaults.
+ * Set `extends: false` to use only your agents and models.
+ *
+ * Supported config file names (in priority order):
+ *   - agent-health.config.ts
+ *   - agent-health.config.js
+ *   - agent-health.config.mjs
+ */
+
+export default {
+  // Custom agents are merged with built-in agents by default.
+  // Each agent `key` must be unique — if it matches a built-in key,
+  // it will override that built-in agent.
+  agents: [
+    // Example 1: REST connector (synchronous JSON response)
+    {
+      key: "my-rest-agent",
+      name: "My REST Agent",
+      endpoint: "http://localhost:8000/api/agent",
+      connectorType: "rest",
+      useTraces: true,           // Enable OpenTelemetry trace collection
+    },
+
+    // Example 2: Streaming connector (Server-Sent Events)
+    {
+      key: "my-streaming-agent",
+      name: "My Streaming Agent",
+      endpoint: "http://localhost:9000/agent/stream",
+      connectorType: "agui-streaming",
+      useTraces: false,
+    },
+
+    // Example 3: CLI tool connector
+    // {
+    //   key: "my-cli-agent",
+    //   name: "My CLI Agent",
+    //   endpoint: "/usr/local/bin/my-agent",
+    //   connectorType: "subprocess",
+    //   useTraces: false,
+    // },
+
+    // Example 4: LangGraph REST connector (non-streaming invoke)
+    // {
+    //   key: "my-langgraph-agent",
+    //   name: "My LangGraph Agent",
+    //   endpoint: "http://localhost:8123",
+    //   connectorType: "langgraph",
+    //   connectorConfig: {
+    //     graphId: "agent",        // Assistant/graph ID (defaults to "agent")
+    //     // threadId: "...",      // Optional thread ID for multi-turn
+    //   },
+    //   useTraces: true,
+    // },
+
+    // Example 5: Amazon Strands connector (Bedrock Agent Runtime)
+    // {
+    //   key: "my-strands-agent",
+    //   name: "My Strands Agent",
+    //   endpoint: "ABCDEF1234",     // Bedrock Agent ID
+    //   connectorType: "strands",
+    //   connectorConfig: {
+    //     agentAliasId: "TSTALIASID",  // Agent alias ID
+    //     region: "us-west-2",          // AWS region (defaults to AWS_REGION)
+    //   },
+    //   useTraces: false,
+    // },
+
+    // Example 6: Agent with authentication hook
+    // {
+    //   key: "authenticated-agent",
+    //   name: "Authenticated Agent",
+    //   endpoint: "https://api.example.com/agent",
+    //   connectorType: "rest",
+    //   useTraces: true,
+    //   hooks: {
+    //     beforeRequest: async ({ endpoint, payload, headers }) => {
+    //       return {
+    //         endpoint,
+    //         payload,
+    //         headers: {
+    //           ...headers,
+    //           'Authorization': `Bearer ${process.env.API_TOKEN}`,
+    //         },
+    //       };
+    //     },
+    //   },
+    // },
+
+    // Example 7: Claude Code with MCP servers (competitive evaluation)
+    // Uses a standard MCP config JSON file (same format as ~/.claude.json mcpServers)
+    // {
+    //   key: "claude-code-eval",
+    //   name: "Claude Code (Eval)",
+    //   endpoint: "claude",
+    //   connectorType: "claude-code",
+    //   useTraces: true,
+    //   connectorConfig: {
+    //     dangerouslySkipPermissions: true,
+    //     appendSystemPrompt: "You are an observability agent...",
+    //     allowedTools: ["mcp__aws-prometheus__*", "mcp__aws-cloudwatch__*", "Bash"],
+    //     mcpConfigPath: "./mcp-config.json",  // standard MCP config file
+    //     strictMcpConfig: true,
+    //     env: { CLAUDE_CODE_USE_BEDROCK: "1", AWS_PROFILE: "Bedrock" },
+    //   },
+    // },
+  ],
+
+  // --- Data sources (OpenSearch clusters) ----------------------------------
+  // storage:       where eval results are persisted (omit for file-based).
+  // observability: where agent traces/logs are read from (Traces tab).
+  // Precedence: agent-health.config.json (Settings UI) -> these TS fields ->
+  //   OPENSEARCH_STORAGE_*/OPENSEARCH_LOGS_* env -> file-based fallback.
+  // Read secrets from process.env so this committed file carries none.
+  //
+  // storage: {
+  //   endpoint: process.env.OPENSEARCH_STORAGE_ENDPOINT,
+  //   authType: 'sigv4',          // 'none' | 'basic' | 'sigv4'
+  //   awsRegion: 'us-east-1',
+  //   awsService: 'es',           // 'es' (managed) | 'aoss' (serverless)
+  //   awsProfile: process.env.AWS_PROFILE,
+  // },
+  //
+  // observability: {
+  //   endpoint: process.env.OPENSEARCH_LOGS_ENDPOINT,
+  //   authType: 'sigv4',
+  //   awsRegion: 'us-east-1',
+  //   indexes: { traces: 'otel-v1-apm-span-*', logs: 'ml-commons-logs-*' },
+  // },
+
+  // Custom models (merged with built-in models by default)
+  // models: [
+  //   {
+  //     key: "my-model",
+  //     model_id: "us.anthropic.claude-sonnet-4-20250514-v1:0",
+  //     display_name: "My Model",
+  //     provider: "bedrock",       // "bedrock" | "openai-compatible" | "demo"
+  //     context_window: 200000,
+  //     max_output_tokens: 4096,
+  //   },
+  // ],
+
+  // Set to false to ignore all built-in agents and models
+  // extends: false,
+};

package/examples/connectors/echo-connector.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Example: a custom connector that wraps a simple HTTP echo agent.
+ *
+ * This is the smallest realistic shape of a connector. It implements the
+ * three abstract members of `BaseConnector` (`buildPayload`, `execute`,
+ * `parseResponse`) plus the three readonly identity fields (`type`,
+ * `name`, `supportsStreaming`).
+ *
+ * Drop this file at `connectors/echo.ts` in your project and register it
+ * from your `agent-health.config.ts`:
+ *
+ * @example
+ * ```ts
+ * import { defineConfig, registerConnector } from '@opensearch-project/agent-health';
+ * import { EchoConnector } from './connectors/echo';
+ *
+ * registerConnector(new EchoConnector());
+ *
+ * export default defineConfig({
+ *   agents: [
+ *     {
+ *       key: 'echo',
+ *       name: 'Echo Agent',
+ *       endpoint: 'http://localhost:8080/echo',
+ *       connectorType: 'rest',  // or any registered protocol
+ *     },
+ *   ],
+ * });
+ * ```
+ *
+ * To understand the lifecycle this connector plugs into — abort signal
+ * handling, progress callbacks, trajectory shape, raw-event replay —
+ * read the base class and types directly in the installed package:
+ *
+ *     node_modules/@opensearch-project/agent-health/lib/dist/services/connectors/base/BaseConnector.js
+ *     node_modules/@opensearch-project/agent-health/lib/dist/services/connectors/types.d.ts
+ *
+ * The compiled `.js` preserves the original JSDoc, and the matching
+ * `.d.ts` next to it gives you the precise type contract — no need to
+ * cross-reference an arbitrary GitHub commit, you're reading exactly
+ * the version your `package.json` resolved to.
+ *
+ * NOTE: `ConnectorProtocol` is a closed string-literal union of the
+ * built-in protocol ids ('rest', 'agui-streaming', 'subprocess', ...).
+ * The simplest path for a custom connector is to reuse one of those ids
+ * (e.g. 'rest' for HTTP-JSON agents) and replace the built-in connector
+ * via `registerConnector(new EchoConnector())`. If you need a brand-new
+ * protocol id, fork this repo and widen the `ConnectorProtocol` union
+ * in `services/connectors/types.ts`.
+ */
+
+import {
+  BaseConnector,
+  type ConnectorAuth,
+  type ConnectorRequest,
+  type ConnectorResponse,
+  type ConnectorProgressCallback,
+  type ConnectorRawEventCallback,
+} from '@opensearch-project/agent-health';
+
+import type { TrajectoryStep } from '@opensearch-project/agent-health';
+
+export class EchoConnector extends BaseConnector {
+  // Replaces the built-in 'rest' connector when registered. Pick a
+  // different id (and widen ConnectorProtocol in your fork) if you
+  // want to coexist with the default REST connector.
+  readonly type = 'rest' as const;
+  readonly name = 'Echo Agent';
+  readonly supportsStreaming = false;
+
+  /**
+   * Build the JSON payload sent to the agent's HTTP endpoint.
+   * Called by `execute()` and also when re-processing stored raw events
+   * during trace replay (see `parseResponse`).
+   */
+  buildPayload(request: ConnectorRequest): { message: string } {
+    return { message: request.testCase.initialPrompt };
+  }
+
+  /**
+   * Single-turn HTTP call → single response.
+   */
+  async execute(
+    endpoint: string,
+    request: ConnectorRequest,
+    _auth: ConnectorAuth,
+    onProgress?: ConnectorProgressCallback,
+    _onRawEvent?: ConnectorRawEventCallback,
+  ): Promise<ConnectorResponse> {
+    const startedAt = Date.now();
+
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(this.buildPayload(request)),
+    });
+    if (!res.ok) {
+      throw new Error(`Echo agent failed: ${res.status} ${res.statusText}`);
+    }
+    const json = (await res.json()) as { reply: string };
+
+    // Build a single-step trajectory and emit progress for live UIs.
+    const step = this.createStep('response', json.reply);
+    onProgress?.(step);
+
+    return {
+      success: true,
+      response: json.reply,
+      trajectory: [step],
+      rawEvents: [{ type: 'ECHO_REPLY', reply: json.reply }],
+      durationMs: Date.now() - startedAt,
+      runId: `echo-${startedAt}`,
+    };
+  }
+
+  /**
+   * Re-derive trajectory from previously stored raw events. Used by
+   * the run replayer / trace recovery path.
+   */
+  parseResponse(rawResponse: unknown): TrajectoryStep[] {
+    const events = Array.isArray(rawResponse) ? rawResponse : [];
+    return events
+      .filter((e: { type?: string }) => e?.type === 'ECHO_REPLY')
+      .map((e: { reply: string }) => this.createStep('response', e.reply));
+  }
+}

package/examples/eval-files/demo.eval.js

@@ -0,0 +1,128 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Demo eval — five test cases that show off the SDK end-to-end against
+ * the live Observio agent.
+ *
+ *   1. observio-says-hello          Pure deterministic, no LLM (free, fast)
+ *   2. observio-uses-a-tool         Trajectory inspection — must invoke at least one tool
+ *   3. observio-rca-is-coherent     Hybrid — deterministic + LLM judge
+ *   4. labels-only-no-prompt        No agent invocation at all (data-only test)
+ *   5. budget-aware                 Uses traces fixture for token-budget assertion
+ *
+ * Run with:
+ *   curl -sN -X POST http://localhost:4002/api/storage/evaluation-runs \
+ *     -H 'Content-Type: application/json' \
+ *     -d '{
+ *       "name":"SDK Demo",
+ *       "sources":[{"type":"code-import","filenames":["evals/demo.eval.js"],"testCaseIds":[]}],
+ *       "agentKey":"observio",
+ *       "modelId":"claude-sonnet"
+ *     }'
+ */
+
+const { test, expect } = require('@opensearch-project/agent-health');
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 1: Pure deterministic — no LLM, $0, fully reproducible
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-says-hello', {
+  prompt: 'Say hello to me in one short sentence.',
+  description: 'Smoke test: agent produces a non-trivial response',
+  labels: ['category:Smoke', 'difficulty:Easy', 'demo', 'agent:observio'],
+}, function ({ result }) {
+  // Trajectory must exist
+  expect(result.trajectory).to.have.length.greaterThan(0);
+
+  // Final output is non-empty
+  expect(result.agentOutput.trim()).to.have.length.greaterThan(0);
+
+  // Reasonable timing budget
+  expect(result).to.haveCompletedWithin(60_000);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 2: Trajectory inspection — proves the agent actually investigates
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-uses-a-tool', {
+  prompt: 'Find the source of the most recent error log entry.',
+  description: 'Agent must call at least one tool, not just answer from memory',
+  labels: ['category:Tool Use', 'difficulty:Medium', 'demo', 'agent:observio'],
+}, function ({ result }) {
+  // At least one action step (tool invocation)
+  expect(result.trajectory).to.haveStepsOfType('action');
+
+  // The trajectory accessor sugar makes the assertion read like prose
+  const tools = result.trajectory.toolCalls();
+  expect(tools).to.have.length.greaterThan(0);
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 3: Hybrid — deterministic preflight + targeted LLM judge
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('observio-rca-is-coherent', {
+  prompt: 'Diagnose why the payment service is failing and explain the root cause.',
+  description: 'Hybrid: cheap structural checks first, then LLM judge for semantic correctness',
+  context: [
+    {
+      description: 'Error log',
+      value: 'ERROR 2024-01-15 10:31:22 [payment-service] Connection refused to database-primary:5432',
+    },
+  ],
+  labels: ['category:RCA', 'difficulty:Hard', 'demo', 'agent:observio', 'hybrid'],
+}, async function ({ result, judge }) {
+  // Cheap deterministic preflight — fail fast, never spend $ on the judge
+  expect(result.trajectory).to.have.length.greaterThan(0);
+  expect(result).to.haveCompletedWithin(120_000);
+  expect(result.agentOutput).to.haveOutputMatching(/payment[- ]service/i);
+
+  // Now invoke the LLM judge for the semantic claim. This produces a
+  // 'hybrid' evaluation method on the resulting report.
+  await judge(result, 'Correctly identifies that the payment-service cannot connect to its database');
+  await judge(result, 'Provides a plausible root-cause hypothesis (e.g. database down, network issue)');
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 4: No prompt at all — pure data-only test, agent never invoked
+// ─────────────────────────────────────────────────────────────────────────────
+//
+// Useful for tests that assert against external state, fixtures, or static
+// data without needing any agent involvement.
+
+test('labels-only-no-prompt', {
+  description: 'Verify a fixture file matches a baseline (no agent involvement)',
+  labels: ['category:Data Quality', 'difficulty:Easy', 'demo'],
+}, function ({ result }) {
+  // No agent ran — durationMs is 0, trajectory empty
+  expect(result.durationMs).to.equal(0);
+  expect(result.trajectory).to.have.length(0);
+
+  // The actual deterministic check (could load a JSON file, compute, etc.)
+  const baseline = { name: 'demo', version: 1 };
+  expect(baseline.version).to.equal(1);
+  expect(baseline.name).to.equal('demo');
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test 5: Budget-aware — uses the traces fixture for token / span timing checks
+// ─────────────────────────────────────────────────────────────────────────────
+//
+// `traces` is pre-loaded by the runner when the agent emits OTel spans. When
+// no traces are available the accessor returns 0 / empty for all helpers,
+// so this test still passes (vacuously) on agents without instrumentation.
+
+test('budget-aware', {
+  prompt: 'Summarize the system in one paragraph.',
+  description: 'Token budget + span-timing assertions via the traces fixture',
+  labels: ['category:Budget', 'difficulty:Easy', 'demo', 'agent:observio'],
+}, function ({ result, traces }) {
+  expect(result.agentOutput.length).to.be.greaterThan(0);
+  // Cheap budget cap — fails if the agent burned more than 50k tokens
+  expect(traces.totalTokens).to.be.lessThan(50_000);
+});

package/examples/eval-files/sdk-hooks-demo.eval.js

@@ -0,0 +1,99 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * SDK lifecycle hooks demo — shows beforeEach/afterEach, beforeAll/afterAll,
+ * and the `provide()` mechanism for surfacing per-test provisioned values
+ * to the body and the teardown hook.
+ *
+ * The example uses a temp directory as the "out-of-band resource" — the
+ * exact thing connectors can't do today (no lifecycle, no teardown). Each
+ * test gets its own directory; concurrent runs are isolated automatically.
+ *
+ * Run with:
+ *   AH_PORT=4002 npx @opensearch-project/agent-health benchmark \
+ *     -f evals/sdk-hooks-demo.eval.js -a demo
+ */
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const {
+  test,
+  beforeAll,
+  afterAll,
+  beforeEach,
+  afterEach,
+  expect,
+} = require('@opensearch-project/agent-health');
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Suite-scoped: runs once before any test, once after them all.
+// ─────────────────────────────────────────────────────────────────────────────
+
+let suiteRoot;
+
+beforeAll(() => {
+  suiteRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ah-hooks-demo-'));
+});
+
+afterAll(() => {
+  if (suiteRoot && fs.existsSync(suiteRoot)) {
+    fs.rmSync(suiteRoot, { recursive: true, force: true });
+  }
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Per-test: each test gets its own subdirectory, surfaced via `provide()`.
+// `afterEach` reads the same key from `provisioned` to clean up, even when
+// the body throws.
+// ─────────────────────────────────────────────────────────────────────────────
+
+beforeEach(({ provide, testInfo }) => {
+  const safe = testInfo.name.replace(/[^a-z0-9-]+/gi, '_');
+  const dir = fs.mkdtempSync(path.join(suiteRoot, `${safe}-`));
+  provide('workspaceDir', dir);
+});
+
+afterEach(({ provisioned }) => {
+  const dir = provisioned.workspaceDir;
+  if (typeof dir === 'string' && fs.existsSync(dir)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Tests — read the provisioned value via `fixtures.provisioned`. No prompt
+// is set so the runner skips the agent and runs the body directly; the
+// hooks still run because they don't depend on the agent step.
+// ─────────────────────────────────────────────────────────────────────────────
+
+test('hook-demo-uses-workspace', {
+  description: 'beforeEach provisioned a fresh temp dir for this test',
+  labels: ['category:Smoke', 'feature:hooks'],
+}, ({ provisioned, testInfo }) => {
+  expect(provisioned.workspaceDir).to.be.a('string');
+  expect(fs.existsSync(provisioned.workspaceDir)).to.equal(true);
+
+  // The dir's basename must include the test name, proving testInfo flowed
+  // through `provide()` correctly.
+  expect(path.basename(provisioned.workspaceDir)).to.include(
+    testInfo.name.replace(/[^a-z0-9-]+/gi, '_')
+  );
+
+  // Write a file inside, just to show it's a real, writable workspace.
+  fs.writeFileSync(path.join(provisioned.workspaceDir, 'marker'), 'hello');
+});
+
+test('hook-demo-isolation', {
+  description: 'A second test gets its own isolated workspace',
+  labels: ['category:Smoke', 'feature:hooks'],
+}, ({ provisioned }) => {
+  expect(provisioned.workspaceDir).to.be.a('string');
+  // Concurrent tests would each see their own dir; this one in particular
+  // must NOT see the marker file from the other test.
+  expect(fs.existsSync(path.join(provisioned.workspaceDir, 'marker'))).to.equal(false);
+});

package/examples/pi-profiling/README.md

@@ -0,0 +1,77 @@
+<!--
+  * Copyright OpenSearch Contributors
+  * SPDX-License-Identifier: Apache-2.0
+-->
+
+# Pi session profiling — drop-in extension
+
+Profile your **pi.dev** coding sessions with Agent Health, the way a CPU profiler
+profiles a running process: instrument a real session, then get back what to fix
+(scored against an evaluator rubric) to improve the agent and your own
+productivity.
+
+pi has **no native OpenTelemetry telemetry**, so `agent-health profile` has
+nothing to read for a live pi session. This single file supplies that missing
+half: it instruments the running session (emitting OTel spans) **and** registers
+the `/agent-health-profile` command. It is **zero-dependency** — it speaks OTLP
+over the built-in `fetch`, so there is nothing to `npm install`.
+
+## Install (out of the box)
+
+```bash
+# 1. Drop the extension in (global = every pi session; or .pi/extensions/ per project)
+mkdir -p ~/.pi/agent/extensions
+cp agent-health-profile.ts ~/.pi/agent/extensions/
+
+# 2. Point telemetry at a trace store BEFORE starting pi.
+#    Simplest: Agent Health in FILE mode (no OpenSearch) — its embedded receiver
+#    persists spans and matches sessions out of the box:
+agent-health serve            # or any command that boots the server on :4001
+export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4001   # /v1/traces is appended
+```
+
+That's it. Use pi normally, then run `/agent-health-profile` (or
+`/agent-health-profile -e <evaluator-id> -f "focus on X"`).
+
+## How it routes (important for OpenSearch users)
+
+The extension exports OTLP/JSON to `OTEL_EXPORTER_OTLP_ENDPOINT`:
+
+- **File mode (recommended for getting started):** point it at Agent Health's
+  embedded `:4001/v1/traces`. Spans persist to disk and `profile` matches them by
+  `session.id` directly. Zero infra.
+- **OpenSearch-backed Agent Health:** the embedded `:4001` receiver *drops*
+  payloads when a cluster is configured (no split-brain). Point the extension at
+  the **same OTLP ingest the cluster reads from** (your OTel Collector / OSIS
+  pipeline). The extension emits a standard OTLP `session.id` attribute — it is
+  schema-agnostic; what matters is that your ingest and `agent-health profile`'s
+  session query agree on where `session.id` lands. Agent Health is standardizing
+  on the plain/nested schema (`attributes.session.id`; OSIS
+  `trace-analytics-plain-raw`), moving off the legacy `@`-flattened
+  `span.attributes.session@id` layout.
+
+## Config (env)
+
+| Var | Default | Meaning |
+|-----|---------|---------|
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4001` | OTLP traces target (`/v1/traces` appended) |
+| `OTEL_SERVICE_NAME` | `pi-agent` | service.name on spans (keep `pi-agent` for `--service` defaults) |
+| `OTEL_ENABLED` | `true` | set `false` to disable telemetry |
+| `AGENT_HEALTH_REDACT` | unset | `1` → redact prompt / tool I/O |
+| `AGENT_HEALTH_CLI` | `npx @opensearch-project/agent-health` | how the command invokes the CLI |
+
+## What it emits
+
+| Span | `gen_ai.operation.name` | Notes |
+|------|-------------------------|-------|
+| `invoke_agent pi` (root) | `invoke_agent` | one per session; carries `session.id` |
+| `chat <model>` (per turn) | `chat` | token usage + `llm.request`/`llm.response` events |
+| `execute_tool <name>` | `execute_tool` | `gen_ai.tool.input`/`output`; ERROR status on failure |
+
+Every span carries `session.id`; the resource carries `service.name=pi-agent`.
+
+## Caveat
+
+Profiling only sees sessions that ran **while the extension was loaded and
+telemetry was on** — you can't profile a session retroactively. Run a session,
+then `/agent-health-profile`.