@opensearch-project/agent-health 0.3.0 → 0.5.2

package/observio-sample-agent/pi-package/extensions/agent-health.ts

@@ -0,0 +1,373 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Agent Health extension for Pi.dev
+ *
+ * Provides tools for:
+ * 1. Validating OTel instrumentation against Gen AI semantic conventions
+ * 2. Checking project compliance (SPDX headers, DCO, changelog)
+ * 3. Running build/test validation
+ * 4. Finding relevant code in the Agent Health architecture
+ */
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { StringEnum } from "@earendil-works/pi-ai";
+
+export default function (pi: ExtensionAPI) {
+  // --- Tool: validate-spans ---
+  pi.registerTool({
+    name: "validate_spans",
+    label: "Validate OTel Spans",
+    description:
+      "Checks if a codebase follows OpenTelemetry Gen AI semantic conventions for agent instrumentation. Scans for span attributes, naming, and hierarchy issues.",
+    promptSnippet: "Validate OTel span instrumentation against Gen AI conventions",
+    promptGuidelines: [
+      "Use validate_spans when the user asks if their instrumentation is correct",
+      "Use validate_spans after generating instrumentation code to verify it",
+      "Use validate_spans when debugging why traces don't appear in Agent Health",
+    ],
+    parameters: Type.Object({
+      path: Type.String({ description: "Path to scan (file or directory)" }),
+      check: StringEnum([
+        "all",
+        "attributes",
+        "naming",
+        "hierarchy",
+        "events",
+      ] as const),
+    }),
+
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const { path, check } = params;
+
+      // Required Gen AI attributes per span type
+      const requirements = {
+        agent: [
+          "gen_ai.operation.name",
+          "gen_ai.system",
+          "gen_ai.agent.name",
+        ],
+        llm: [
+          "gen_ai.operation.name",
+          "gen_ai.system",
+          "gen_ai.request.model",
+          "gen_ai.usage.input_tokens",
+          "gen_ai.usage.output_tokens",
+        ],
+        tool: ["gen_ai.operation.name", "gen_ai.tool.name"],
+      };
+
+      // Required span events
+      const requiredEvents = {
+        llm: ["gen_ai.content.prompt", "gen_ai.content.completion"],
+        tool: ["gen_ai.tool.input", "gen_ai.tool.output"],
+      };
+
+      // Expected operation names
+      const validOps = ["invoke_agent", "chat", "execute_tool"];
+
+      // Expected span naming: "<operation> <system/tool>"
+      const namingPattern = /^(invoke_agent|chat|execute_tool)\s+\S+/;
+
+      const report = {
+        path,
+        check,
+        requirements,
+        requiredEvents,
+        validOps,
+        namingPattern: namingPattern.source,
+        instructions: [
+          `Scan files at '${path}' for OpenTelemetry span creation calls`,
+          "Check startSpan() calls for required attributes based on span type",
+          "Verify span names follow '<operation> <system>' pattern",
+          "Check for addEvent() calls with required event names",
+          "Verify parent-child hierarchy: agent → llm/tool spans",
+          "Report any missing attributes, bad naming, or hierarchy issues",
+        ],
+      };
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(report, null, 2),
+          },
+        ],
+      };
+    },
+  });
+
+  // --- Tool: check-compliance ---
+  pi.registerTool({
+    name: "check_compliance",
+    label: "Check PR Compliance",
+    description:
+      "Validates Agent Health project compliance: SPDX license headers, DCO signoff, CHANGELOG entries, and build status.",
+    promptSnippet: "Check PR compliance (SPDX, DCO, changelog, build)",
+    promptGuidelines: [
+      "Use check_compliance before creating a PR to catch issues early",
+      "Use check_compliance when the user asks about PR requirements",
+    ],
+    parameters: Type.Object({
+      check: StringEnum([
+        "all",
+        "spdx",
+        "dco",
+        "changelog",
+        "build",
+      ] as const),
+    }),
+
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const checks: Record<string, object> = {};
+
+      if (params.check === "all" || params.check === "spdx") {
+        checks.spdx = {
+          description: "All source files need SPDX Apache-2.0 header",
+          header:
+            "/* Copyright OpenSearch Contributors\n SPDX-License-Identifier: Apache-2.0 */",
+          check: "grep -rL 'SPDX-License-Identifier' src/ --include='*.ts'",
+        };
+      }
+
+      if (params.check === "all" || params.check === "dco") {
+        checks.dco = {
+          description: "All commits need DCO signoff",
+          check: "git log --no-merges --format='%H %s' | head -5",
+          fix: "git commit -s -m 'message'",
+          note: "Use -s flag, never --no-verify",
+        };
+      }
+
+      if (params.check === "all" || params.check === "changelog") {
+        checks.changelog = {
+          description: "CHANGELOG.md must be updated for PRs",
+          check: "git diff main -- CHANGELOG.md",
+          format: "- Description of change ([#PR](link)) by @author",
+        };
+      }
+
+      if (params.check === "all" || params.check === "build") {
+        checks.build = {
+          description: "Build and all tests must pass",
+          command: "npm run build:all && npm run test:all",
+          note: "Never use --no-verify to skip hooks",
+        };
+      }
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(checks, null, 2),
+          },
+        ],
+      };
+    },
+  });
+
+  // --- Tool: find-architecture ---
+  pi.registerTool({
+    name: "find_architecture",
+    label: "Find Relevant Code",
+    description:
+      "Routes a query to the correct layer in Agent Health's architecture. Knows where services, routes, storage, CLI commands, types, and UI components live.",
+    promptSnippet: "Find relevant code in Agent Health's architecture",
+    promptGuidelines: [
+      "Use find_architecture when implementing a feature to find the right files",
+      "Use find_architecture when the user asks where something is implemented",
+    ],
+    parameters: Type.Object({
+      area: StringEnum([
+        "agent-streaming",
+        "evaluation",
+        "storage",
+        "traces",
+        "cli",
+        "routes",
+        "types",
+        "config",
+        "ui-components",
+        "connectors",
+      ] as const),
+      query: Type.Optional(
+        Type.String({ description: "What specifically to find" })
+      ),
+    }),
+
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const map: Record<string, object> = {
+        "agent-streaming": {
+          dir: "services/agent/",
+          key_files: ["sseHandler.ts", "agUiConverter.ts"],
+          patterns: ["SSE streaming", "AG-UI protocol", "event-stream headers"],
+        },
+        evaluation: {
+          dir: "services/evaluation/",
+          routes: "server/routes/judge.ts",
+          key_files: ["judge.ts", "mockEvaluator.ts"],
+          patterns: ["Bedrock judge", "passFailStatus", "accuracy"],
+        },
+        storage: {
+          dir: "services/storage/",
+          adapter: "server/adapters/opensearch/StorageModule.ts",
+          key_files: [
+            "asyncTestCaseStorage.ts",
+            "asyncBenchmarkStorage.ts",
+            "asyncRunStorage.ts",
+          ],
+          patterns: [
+            "toAppFormat/toStorageFormat",
+            "CRUD wrappers",
+            "OpenSearch client",
+          ],
+        },
+        traces: {
+          dir: "services/traces/",
+          key_files: [
+            "spanCategorization.ts",
+            "metrics.ts",
+            "tracePoller.ts",
+          ],
+          patterns: [
+            "Gen AI semantic conventions",
+            "token/cost calculation",
+            "background polling",
+          ],
+        },
+        cli: {
+          dir: "cli/",
+          key_files: [
+            "commands/run.ts",
+            "commands/benchmark.ts",
+            "commands/export.ts",
+            "utils/serverLifecycle.ts",
+            "utils/apiClient.ts",
+          ],
+          patterns: [
+            "ensureServer()",
+            "server auto-start",
+            "typed HTTP client",
+          ],
+        },
+        routes: {
+          dir: "server/routes/",
+          patterns: [
+            "validateX(): string | null",
+            "return 400 with { error }",
+            "early return for 404",
+          ],
+        },
+        types: {
+          file: "types/index.ts",
+          key_types: [
+            "TestCase",
+            "BenchmarkRun",
+            "TrajectoryStep",
+            "AgentConfig",
+          ],
+          patterns: [
+            "string unions not enums",
+            "discriminated unions with type field",
+          ],
+        },
+        config: {
+          dir: "lib/",
+          key_files: [
+            "config/loader.ts",
+            "config/types.ts",
+            "constants.ts",
+            "hooks.ts",
+          ],
+          patterns: [
+            "agent-health.config.ts",
+            "UserAgentConfig",
+            "beforeRequest hook",
+          ],
+        },
+        "ui-components": {
+          dir: "components/",
+          patterns: ["shadcn/ui", "React hooks in hooks/"],
+        },
+        connectors: {
+          dir: "services/connectors/",
+          patterns: [
+            "ConnectorType",
+            "agui-streaming",
+            "custom connector template",
+          ],
+          skill: "/add-connector for full guide",
+        },
+      };
+
+      const result = map[params.area] || { error: "Unknown area" };
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(
+              { area: params.area, query: params.query, ...result },
+              null,
+              2
+            ),
+          },
+        ],
+      };
+    },
+  });
+
+  // --- Tool: run-validation ---
+  pi.registerTool({
+    name: "run_validation",
+    label: "Run Build & Tests",
+    description:
+      "Runs the Agent Health build and test suite. Use before creating PRs.",
+    promptSnippet: "Run npm build:all && test:all for Agent Health",
+    promptGuidelines: [
+      "Use run_validation after implementing changes to verify nothing is broken",
+      "Use run_validation before creating a PR",
+    ],
+    parameters: Type.Object({
+      scope: StringEnum([
+        "all",
+        "build",
+        "unit",
+        "integration",
+        "e2e",
+      ] as const),
+    }),
+
+    async execute(toolCallId, params, signal, onUpdate, ctx) {
+      const commands: Record<string, string> = {
+        all: "npm run build:all && npm run test:all",
+        build: "npm run build:all",
+        unit: "npm run test:unit",
+        integration: "npm run test:integration",
+        e2e: "npm run test:e2e",
+      };
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({
+              command: commands[params.scope],
+              note: "Run this in the agent-health project root",
+              coverage_thresholds: {
+                lines: "90%",
+                statements: "90%",
+                functions: "80%",
+                branches: "80%",
+              },
+            }),
+          },
+        ],
+      };
+    },
+  });
+}

package/observio-sample-agent/pi-package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@opensearch-project/agent-health-pi",
+  "version": "0.1.0",
+  "description": "Pi.dev coding agent for Agent Health — helps instrument agents with OTel and contribute to the project",
+  "keywords": ["pi-package"],
+  "license": "Apache-2.0",
+  "pi": {
+    "extensions": ["./extensions"],
+    "skills": ["./skills"],
+    "prompts": ["./prompts"]
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-agent-core": "*",
+    "@earendil-works/pi-coding-agent": "*"
+  }
+}

package/observio-sample-agent/pi-package/prompts/agent-health.md

@@ -0,0 +1,37 @@
+# Agent Health Coding Assistant
+
+You are a specialized coding agent for the **Agent Health** project — an evaluation and observability framework for AI agents built on OpenSearch.
+
+## Your Two Modes
+
+### 1. DIY Observability (for customers with their own agents)
+Help users instrument their AI agents with OpenTelemetry so traces flow into Agent Health. You know:
+- Gen AI semantic conventions (`gen_ai.operation.name`, `gen_ai.system`, etc.)
+- How Agent Health categorizes spans (AGENT/LLM/TOOL/FRAMEWORK)
+- OTLP export setup (OSIS pipelines, OTel Collector, direct)
+- How to verify traces appear correctly
+
+Use skills: `/instrument-otel`, `/setup-collector`
+
+### 2. Contributor Mode (implementing features and fixes)
+Help anyone contribute to Agent Health itself. You know:
+- The architecture: server-mediated CLI, service layers, storage adapters
+- Coding conventions: SSE streaming, cancellation tokens, route validation, `@/` imports
+- PR requirements: SPDX headers, DCO signoff, CHANGELOG, test coverage
+
+Use skills: `/implement-feature`, `/fix-bug`, `/create-pr`, `/write-test`
+
+## Key Principles
+
+- **Read before writing.** Always examine existing code in the relevant directory before implementing.
+- **Minimal changes.** Fix what's asked, don't refactor the neighborhood.
+- **Architecture-aware.** Route work to the correct layer. CLI calls server, server calls storage. Never bypass.
+- **Test everything.** 90% line coverage, `@/` imports, mock at boundaries only.
+- **Compliance always.** SPDX headers, DCO signoff, CHANGELOG updates. Never skip hooks.
+
+## Tools Available
+
+- `validate_spans` — Check OTel instrumentation correctness
+- `check_compliance` — Verify PR readiness (SPDX, DCO, changelog, build)
+- `find_architecture` — Route queries to the right code layer
+- `run_validation` — Run build and test suite

package/observio-sample-agent/pi-package/skills/create-pr/SKILL.md

@@ -0,0 +1,88 @@
+---
+name: create-pr
+description: >
+  Create a pull request for Agent Health following all compliance requirements.
+  Use after implementing a feature or fix. Handles SPDX headers, DCO signoff,
+  CHANGELOG, and the PR template.
+---
+
+# Create Pull Request for Agent Health
+
+You are helping the user create a compliant PR for the Agent Health project.
+
+## Pre-flight Checks
+
+Run these before creating the PR:
+
+### 1. Build & Tests
+```bash
+npm run build:all && npm run test:all
+```
+Must pass. Never use `--no-verify`.
+
+### 2. SPDX License Headers
+Every new `.ts`, `.tsx`, `.js` source file needs:
+```
+/* Copyright OpenSearch Contributors
+ SPDX-License-Identifier: Apache-2.0 */
+```
+
+Check for missing headers:
+```bash
+grep -rL 'SPDX-License-Identifier' src/ cli/ server/ services/ --include='*.ts' --include='*.tsx'
+```
+
+### 3. DCO Signoff
+Every commit needs `Signed-off-by:` line:
+```bash
+git log --format='%H %s' origin/main..HEAD | while read hash msg; do
+  git log -1 --format='%b' $hash | grep -q 'Signed-off-by:' || echo "MISSING: $hash $msg"
+done
+```
+
+Fix missing signoff (for last commit):
+```bash
+git commit --amend -s --no-edit
+```
+
+### 4. CHANGELOG.md
+Add entry under `## [Unreleased]` in the appropriate section:
+- `### Added` — new features
+- `### Fixed` — bug fixes
+- `### Changed` — modifications to existing features
+- `### Removed` — removed features
+
+Format: `- Description of change ([#PR](link))`
+
+## Create the PR
+
+```bash
+gh pr create --title "<type>: <concise description>" --body "$(cat <<'EOF'
+## Summary
+- <bullet 1>
+- <bullet 2>
+
+## Test plan
+- [ ] Unit tests added/updated
+- [ ] `npm run build:all && npm run test:all` passes
+- [ ] Manual verification: <describe what to check>
+
+EOF
+)"
+```
+
+### Title Conventions
+- `feat:` — new feature
+- `fix:` — bug fix
+- `refactor:` — code restructuring (no behavior change)
+- `docs:` — documentation only
+- `test:` — test additions/changes
+- `chore:` — maintenance tasks
+
+Keep title under 70 characters.
+
+## After PR Creation
+
+- CI will run: tests, coverage, DCO check, changelog check, link checker
+- Address review feedback with new commits (don't force-push unless asked)
+- Ensure all CI checks pass before requesting re-review

package/observio-sample-agent/pi-package/skills/fix-bug/SKILL.md

@@ -0,0 +1,71 @@
+---
+name: fix-bug
+description: >
+  Diagnose and fix bugs in Agent Health. Use when a user reports a bug, failing test,
+  or unexpected behavior. Follows a systematic diagnosis → fix → verify workflow.
+---
+
+# Fix Bug in Agent Health
+
+You are helping the user diagnose and fix a bug in Agent Health.
+
+## Step 1: Reproduce & Locate
+
+1. **Understand the symptom**: What's the expected vs actual behavior?
+2. **Identify the layer**: Use `find_architecture` tool to narrow down where the bug likely lives
+3. **Find the code path**: Trace from the trigger (UI action, CLI command, API call) through the layers
+
+### Common Bug Locations by Symptom
+
+| Symptom | Likely Location |
+|---------|----------------|
+| Traces not appearing | `services/traces/tracePoller.ts`, `spanCategorization.ts` |
+| Wrong token/cost metrics | `services/traces/metrics.ts` |
+| Storage errors | `services/storage/async*Storage.ts`, `server/adapters/` |
+| CLI command fails | `cli/commands/`, `cli/utils/apiClient.ts` |
+| SSE stream drops | `services/agent/sseHandler.ts`, response flushing |
+| Evaluation incorrect | `server/routes/judge.ts`, prompt templates |
+| UI not updating | `hooks/`, `services/client/` |
+| Agent connector fails | `services/connectors/`, connector-specific adapter |
+
+## Step 2: Diagnose
+
+Read the relevant code. Look for:
+- Off-by-one errors in iteration/pagination
+- Missing null checks at system boundaries (API responses, storage queries)
+- Race conditions in async flows (SSE, polling, cancellation)
+- Type mismatches between storage format and app format
+- Incorrect OpenSearch query construction
+
+**Do NOT guess.** Read the code, understand the flow, then identify the root cause.
+
+## Step 3: Fix
+
+Apply the minimal fix:
+- Fix the actual bug, don't refactor surrounding code
+- Don't add defensive error handling for scenarios that can't happen
+- Don't add comments explaining the fix (the commit message does that)
+- If the fix touches a type, check all usages of that type
+
+## Step 4: Write a Regression Test
+
+Every bug fix should have a test that would have caught it:
+- Test the specific scenario that was broken
+- Use `@/` path alias
+- Place in `tests/` mirroring the source path
+
+## Step 5: Verify
+
+1. Run `run_validation` tool (scope: "all")
+2. Manually verify the original symptom is resolved
+3. Check that no other tests broke (the fix didn't introduce regressions)
+
+## Step 6: Commit & PR
+
+```bash
+git commit -s -m "fix: <concise description of what was wrong>
+
+<one line explaining root cause>"
+```
+
+Update CHANGELOG.md under the appropriate section.