@indemn/cli 1.0.0 → 1.1.1

package/package.json

@@ -1,38 +1,48 @@
 {
   "name": "@indemn/cli",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "description": "CLI, MCP server, and plugin for managing Indemn AI agents",
   "type": "module",
   "main": "./dist/sdk/index.js",
   "bin": {
-    "indemn": "./dist/cli/index.js"
+    "indemn": "dist/cli/index.js"
   },
   "scripts": {
     "build": "tsc",
+    "postbuild": "cp -r src/report/assets dist/report/assets",
     "dev": "tsx --watch src/cli/index.ts",
     "test": "vitest run",
     "test:watch": "vitest"
   },
-  "keywords": ["ai-agents", "insurance", "evaluations", "indemn", "mcp"],
+  "keywords": [
+    "ai-agents",
+    "insurance",
+    "evaluations",
+    "indemn",
+    "mcp"
+  ],
   "author": "Indemn <support@indemn.com>",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/indemn-ai/indemn-cli"
+    "url": "git+https://github.com/indemn-ai/indemn-cli.git"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.25.1",
+    "@react-pdf/renderer": "^4.3.2",
     "chalk": "^5.4.1",
     "cli-table3": "^0.6.5",
     "commander": "^13.1.0",
     "conf": "^13.1.0",
     "firebase": "^11.6.0",
     "ora": "^8.2.0",
+    "react": "^19.2.4",
     "socket.io-client": "^4.8.1",
     "zod": "^3.24.0"
   },
   "devDependencies": {
     "@types/node": "^22.15.0",
+    "@types/react": "^19.2.14",
     "tsx": "^4.19.0",
     "typescript": "^5.9.3",
     "vitest": "^3.1.0"

package/skills/eval-analysis/SKILL.md

@@ -0,0 +1,178 @@
+---
+name: eval-analysis
+description: Analyze evaluation results to classify failures as agent issues vs evaluation issues. Also export conversation traces for sharing. Use when the user asks to analyze eval results, understand why tests failed, triage failures, improve evaluation scores, pull conversation traces, or share eval results.
+argument-hint: [run-id or agent-id]
+allowed-tools: Bash(indemn *)
+---
+
+# Eval Analysis
+
+Analyze evaluation run results, classify each failure, and recommend fixes. Also supports exporting conversation traces and generating branded PDF reports.
+
+**Input:** `$ARGUMENTS` — a run_id (UUID) or agent_id (MongoDB ObjectId)
+
+## Step 1: Resolve the Run
+
+Determine what `$ARGUMENTS` is:
+- **UUID format** (contains hyphens, 36 chars) → it's a `run_id`. Use directly.
+- **ObjectId format** (24 hex chars) → it's an `agent_id`. Fetch the latest completed run:
+  ```bash
+  indemn eval list --agent-id $ARGUMENTS --json --limit 5
+  ```
+  Extract the latest `run_id` with `status: completed`.
+
+If no completed runs exist, tell the user and stop.
+
+## Step 2: Pull All Data
+
+Fetch all data using the CLI (parallelize where possible):
+
+```bash
+# Run summary
+indemn eval status <run-id> --json
+
+# Per-item results
+indemn eval results <run-id> --json
+
+# Rubric (get rubric_id from run summary)
+indemn rubric get <rubric-id> --json
+
+# Test set (get test_set_id from run summary)
+indemn testset get <test-set-id> --json
+
+# Bot context (get agent_id from run summary)
+indemn eval bot-context <agent-id> --json
+```
+
+For exact response shapes, see `references/data-shapes.md`.
+
+## Step 3: Present Run Overview
+
+Show the user a summary table:
+
+```
+Run: {run_id}
+Agent: {bot_name} ({agent_id})
+LLM: {bot_llm_provider}/{bot_llm_model}
+Status: {passed}/{total} items passed ({pass_rate}%)
+
+Criteria: {criteria_passed}/{criteria_total} checks passed
+Rubric: {rubric_rules_passed}/{rubric_rules_total} rules passed
+
+Component Scores:
+  prompt:         {score}% ({passed}/{total})
+  knowledge_base: {score}% ({passed}/{total})
+  function:       {score}% ({passed}/{total})
+  general:        {score}% ({passed}/{total})
+```
+
+## Step 4: Analyze Each Failed Item
+
+For every result where `passed == false`:
+
+1. **Get the test item** — match `test_case_id` to the test set's `items[].item_id` to get the item name, type, inputs, and success_criteria
+2. **Check criteria scores** — from `criteria_scores[]`, list which criteria passed and which failed with their reasoning
+3. **Check rubric scores** — from `rubric_scores[]`, list which rules passed and which failed with their reasoning
+4. **Check for divergence** — flag when criteria say FAIL but rubric says PASS (or vice versa). Divergence is a strong signal of an evaluation problem.
+5. **Read the conversation** — look at `output` field for the full agent response or conversation transcript
+6. **Cross-reference system prompt** — check if the agent's behavior aligns with what the system prompt instructs
+7. **Classify** — assign one of four categories (see below)
+
+### Failure Categories
+
+| Category | Signal | Action |
+|----------|--------|--------|
+| **Real Agent Problem** | Both criteria and rubric agree agent misbehaved, or output contradicts system prompt | Investigate agent code/prompt |
+| **Evaluation Problem** | Criteria ambiguous/too strict, evaluator reasoning shows misunderstanding, agent response was reasonable | Update test set criteria |
+| **Test Design Problem** | Test assumptions wrong (e.g., assumed KB lacked info it has), criteria contradict agent's actual capabilities | Redesign test item |
+| **KB/Retrieval Problem** | Agent behavior correct but retrieval returned wrong content, or KB is missing needed info | Fix KB content or retrieval config |
+
+### Divergence Analysis
+
+When `criteria_passed != rubric_passed` for an item, this is a first-class finding:
+- **Criteria FAIL + Rubric PASS** → criteria are likely too strict or testing the wrong thing
+- **Criteria PASS + Rubric FAIL** → rubric may have rules that don't apply to this scenario, or agent violated a behavioral rule despite answering correctly
+
+Always highlight divergences prominently in the report.
+
+## Step 5: Aggregate Findings
+
+Group all failed items by category:
+
+```
+## Findings
+
+### Real Agent Problems (N items)
+- Item X: [description of what went wrong]
+
+### Evaluation Problems (N items)
+- Item Y: [why the criteria/rubric is wrong]
+
+### Test Design Problems (N items)
+- Item Z: [what assumption was incorrect]
+
+### KB/Retrieval Problems (N items)
+- Item W: [what retrieval issue occurred]
+```
+
+## Step 6: Recommendations
+
+Prioritize fixes:
+
+- **P1 — Evaluation fixes** (false failures inflate failure count, masking real issues): specific criteria rewrites or rubric rule adjustments
+- **P2 — Agent investigation** (real bugs): which code paths or prompt sections to examine
+- **P3 — KB/Retrieval fixes**: which knowledge bases need content updates or which retrieval configs need tuning
+
+For each recommendation, be specific: quote the current criterion/rule, explain what's wrong, and propose the exact replacement text.
+
+## Step 7: Offer to Execute Fixes
+
+Ask the user if they want to apply the evaluation fixes. If yes:
+
+**Update test set criteria** (creates a new version, non-destructive):
+```bash
+# Save the updated test set to a JSON file, then:
+indemn testset update <test-set-id> --file updated-testset.json
+```
+
+**Update rubric rules** (creates a new version, non-destructive):
+```bash
+# Save the updated rubric to a JSON file, then:
+indemn rubric update <rubric-id> --file updated-rubric.json
+```
+
+Both update commands create new versions — the previous version is preserved and accessible via `indemn testset versions <id>` / `indemn rubric versions <id>`.
+
+After applying fixes, suggest re-running the evaluation:
+```bash
+indemn eval run <agent-id> --test-set <test-set-id> --rubric <rubric-id> --wait
+```
+
+## Export: Conversation Traces
+
+When the user wants to **share results** or **export conversation traces** (rather than triage failures):
+
+### Markdown Export
+```bash
+indemn eval export <run-id>
+# Optional: specify output path
+indemn eval export <run-id> --output results.md
+```
+
+Produces a structured markdown file with all conversations, criteria scores, and rubric scores.
+
+### PDF Report
+```bash
+indemn eval report <run-id>
+# Optional: specify output path
+indemn eval report <run-id> --output report.pdf
+```
+
+Produces a branded Indemn PDF with cover page, dual scores (criteria + rubric), and per-item detail pages.
+
+### Agent Card
+```bash
+indemn agents card <agent-id>
+```
+
+Produces a multi-page branded PDF with system prompt, tools, KBs, LLM config, evaluation history, and metadata.

package/skills/eval-analysis/references/data-shapes.md

@@ -0,0 +1,279 @@
+# Evaluation API Data Shapes
+
+Response shapes from the evaluations API at `localhost:8002`. All endpoints prefixed with `/api/v1`.
+
+## EvaluationRun
+
+**Endpoint:** `GET /runs/{run_id}`
+
+```json
+{
+  "run_id": "uuid",
+  "dataset_id": "uuid",
+  "agent_id": "mongodb-objectid",
+  "status": "pending | running | completed | failed",
+  "total": 13,
+  "completed": 13,
+  "passed": 8,
+  "failed": 5,
+  "concurrency": 3,
+  "started_at": "datetime",
+  "completed_at": "datetime",
+  "error": "string | null",
+  "rubric_id": "uuid | null",
+  "rubric_version": 1,
+  "test_set_id": "uuid | null",
+  "test_set_version": 1,
+  "component_scope_filter": "string | null",
+  "component_ids_filter": ["string"] ,
+  "component_scores": {
+    "prompt": { "score": 0.85, "total": 20, "passed": 17 },
+    "knowledge_base": { "score": 0.75, "total": 8, "passed": 6 },
+    "function": { "score": 1.0, "total": 4, "passed": 4 },
+    "general": { "score": 0.9, "total": 10, "passed": 9 }
+  },
+  "criteria_passed": 35,
+  "criteria_total": 40,
+  "rubric_rules_passed": 90,
+  "rubric_rules_total": 100,
+  "bot_llm_provider": "openai",
+  "bot_llm_model": "gpt-4.1",
+  "created_at": "datetime"
+}
+```
+
+**Key fields:**
+- `component_scores` — rubric rule pass rates broken down by component scope (prompt, knowledge_base, function, general). Null if no rubric used.
+- `criteria_passed/total` — aggregate criterion-level pass counts across ALL results
+- `rubric_rules_passed/total` — aggregate rubric rule pass counts across ALL results
+- `test_set_id` — present for V2 runs. `dataset_id` is for V1 question-set-based runs.
+
+## EvaluationResult
+
+**Endpoint:** `GET /runs/{run_id}/results`
+
+Returns an array of results, one per test item.
+
+```json
+{
+  "result_id": "uuid",
+  "run_id": "uuid",
+  "test_case_id": "uuid (matches TestItem.item_id)",
+  "langsmith_run_id": "uuid | null",
+  "input": {
+    "persona": "...",
+    "initial_message": "...",
+    "max_turns": 8
+  },
+  "output": {
+    "messages": [
+      { "role": "user", "content": "..." },
+      { "role": "assistant", "content": "..." }
+    ]
+  },
+  "scores": {
+    "evaluator_name": { "score": 0.8, "comment": "JSON string with details" }
+  },
+  "item_name": "PTO Policy Inquiry",
+  "item_type": "scenario",
+  "criteria_scores": [
+    {
+      "criterion": "Agent retrieves PTO policy from handbook KB",
+      "passed": false,
+      "score": 0.0,
+      "reasoning": "Agent did not find PTO information..."
+    }
+  ],
+  "criteria_passed": false,
+  "rubric_scores": [
+    {
+      "rule_id": "BRIEF_RESPONSES",
+      "rule_name": "Brief, Focused Responses",
+      "severity": "medium",
+      "passed": true,
+      "score": 1.0,
+      "reasoning": "Agent kept responses concise..."
+    }
+  ],
+  "rubric_passed": true,
+  "passed": false,
+  "duration_ms": 45000,
+  "created_at": "datetime"
+}
+```
+
+**V1 vs V2 scoring:**
+- **V1** (question-set runs): `scores` dict populated, `criteria_scores`/`rubric_scores` are null. The `comment` field in scores is a JSON string — parse it for per-criterion details.
+- **V2** (test-set runs): `criteria_scores[]` and `rubric_scores[]` populated with structured data. `scores` may still be present.
+
+**Passed logic:** `passed = criteria_passed AND rubric_passed`. An item fails if ANY criterion fails OR any high-severity rubric rule fails.
+
+## Rubric
+
+**Endpoint:** `GET /rubrics/{rubric_id}`
+
+```json
+{
+  "rubric_id": "uuid",
+  "agent_id": "mongodb-objectid",
+  "version": 1,
+  "name": "O'Connor Internal Ops Associate Rubric",
+  "description": "...",
+  "severity_definitions": {
+    "high": "Critical violation...",
+    "medium": "Moderate violation...",
+    "low": "Minor violation..."
+  },
+  "rules": [
+    {
+      "id": "BRIEF_RESPONSES",
+      "name": "Brief, Focused Responses",
+      "severity": "medium",
+      "category": "response_quality",
+      "description": "Agent should provide concise answers...",
+      "prompt_reference": "Section: Communication Style",
+      "evaluation_criteria": {
+        "pass_conditions": ["Response is under 3 sentences for simple questions"],
+        "fail_conditions": ["Response includes unnecessary preamble or filler"]
+      },
+      "examples": {
+        "violation": "...",
+        "correct": "..."
+      },
+      "component_scope": "prompt | knowledge_base | function | general | null",
+      "component_ids": ["objectid", "..."],
+      "component_names": ["HR Handbook KB"]
+    }
+  ],
+  "created_at": "datetime",
+  "updated_at": "datetime"
+}
+```
+
+**Key fields:**
+- `rules[].evaluation_criteria` — pass/fail conditions the evaluator LLM uses to judge
+- `rules[].severity` — high severity failures cause the entire item to fail
+- `rules[].component_scope` — links rule to a specific component type for component-level scoring
+- `rules[].component_ids` / `component_names` — links rule to specific bot components (KBs, tools, etc.)
+
+## TestSet
+
+**Endpoint:** `GET /test-sets/{test_set_id}`
+
+```json
+{
+  "test_set_id": "uuid",
+  "agent_id": "mongodb-objectid",
+  "version": 2,
+  "name": "O'Connor Internal Ops Associate Test Set",
+  "description": "...",
+  "items": [
+    {
+      "item_id": "uuid (matches EvaluationResult.test_case_id)",
+      "type": "single_turn | scenario",
+      "name": "PTO Policy Inquiry",
+      "inputs": {
+        "message": "What is our PTO policy?",
+        "persona": "An employee asking about...",
+        "initial_message": "Hi, I need to know...",
+        "max_turns": 8
+      },
+      "expected": {
+        "success_criteria": [
+          "Agent retrieves PTO policy from knowledge base",
+          "Response includes accrual rates or refers to handbook"
+        ],
+        "should_use_tools": ["search_kb"],
+        "should_not_use_tools": ["live_handoff"],
+        "expected_outcome": "Employee gets PTO info"
+      },
+      "tags": ["hr", "policy"],
+      "priority": "high"
+    }
+  ],
+  "created_at": "datetime",
+  "updated_at": "datetime"
+}
+```
+
+**Key fields:**
+- `items[].item_id` — join key to `EvaluationResult.test_case_id`
+- `items[].type` — `single_turn` sends one message; `scenario` runs multi-turn simulation with persona
+- `items[].inputs` — for single_turn: `message`; for scenario: `persona`, `initial_message`, `max_turns`
+- `items[].expected.success_criteria` — the criteria that become `criteria_scores[]` in results
+
+## BotContext
+
+**Endpoint:** `GET /bots/{agent_id}/context`
+
+```json
+{
+  "bot_id": "mongodb-objectid",
+  "bot_name": "O'Connor Internal Operations Associate",
+  "organization_id": "mongodb-objectid",
+  "organization_name": "O'Connor Insurance",
+  "system_prompt": "You are an AI assistant for O'Connor Insurance...",
+  "llm_config": {
+    "provider": "openai",
+    "model": "gpt-4.1",
+    "parameters": { "temperature": 0.3 }
+  },
+  "tools": [
+    {
+      "id": "objectid",
+      "name": "Live Chat Handoff",
+      "tool_name": "live_handoff",
+      "type": "human_handoff",
+      "description": "Transfer to a live agent when...",
+      "output_instructions": "..."
+    }
+  ],
+  "knowledge_bases": [
+    {
+      "id": "objectid",
+      "name": "O'Connor Employee Handbook",
+      "type": "file",
+      "document_count": 3
+    }
+  ],
+  "harness_context": {
+    "version": "1.0",
+    "description": "LangGraph-based agent...",
+    "execution_flow": [
+      { "step": 1, "node": "invoke_model", "description": "..." }
+    ],
+    "key_behaviors": ["..."],
+    "evaluation_notes": ["..."]
+  }
+}
+```
+
+**Key fields:**
+- `system_prompt` — the full agent prompt; cross-reference against agent behavior to distinguish real bugs from expected behavior
+- `tools[]` — available tools with their descriptions; critical for understanding handoff, search, and function-calling behavior
+- `knowledge_bases[]` — connected KBs; check if test items assume KB content that may not exist
+- `harness_context` — how bot-service executes the agent; useful for understanding tool call flow
+
+## Versioning Endpoints
+
+Both test sets and rubrics support version history:
+
+```bash
+# List all versions
+GET /test-sets/{test_set_id}/versions
+GET /rubrics/{rubric_id}/versions
+
+# Get a specific version
+GET /test-sets/{test_set_id}/versions/{version}
+GET /rubrics/{rubric_id}/versions/{version}
+```
+
+PUT to update creates a new version automatically — previous versions are preserved.
+
+## Listing Runs for an Agent
+
+```bash
+GET /runs?agent_id={agent_id}&status=completed&limit=5
+```
+
+Returns runs sorted by `created_at` descending. Use this when given an agent_id instead of a run_id.

package/src/cli/agents.ts

@@ -2,6 +2,7 @@ import { Command } from 'commander';
 import ora from 'ora';
 import { IndemnClient } from '../sdk/client.js';
 import { AgentsSDK } from '../sdk/agents.js';
+import { exportAgentCard } from '../report/export.js';
 import { formatTable, printJson, printError, printSuccess, confirm } from './utils/output.js';
 
 function formatChannels(channels: unknown[]): string {
@@ -178,6 +179,27 @@ export function registerAgentCommands(program: Command): void {
       }
     });
 
+  agents
+    .command('card')
+    .argument('<id>', 'Agent ID')
+    .description('Generate branded agent card PDF')
+    .option('--output <path>', 'Output file path')
+    .action(async (id: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const spinner = ora('Fetching agent data...').start();
+
+        spinner.text = 'Generating PDF...';
+        const result = await exportAgentCard(id, options.output, client);
+
+        spinner.stop();
+        printSuccess(`Agent card saved to ${result.file_path}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
   agents
     .command('clone')
     .argument('<id>', 'Agent ID to clone')

package/src/cli/eval.ts

@@ -6,6 +6,7 @@ import { IndemnClient } from '../sdk/client.js';
 import { EvalSDK } from '../sdk/eval.js';
 import { RubricsSDK } from '../sdk/rubrics.js';
 import { TestSetsSDK } from '../sdk/test-sets.js';
+import { exportEvalMarkdown, exportEvalReport } from '../report/export.js';
 import { formatTable, printJson, printError, printSuccess, confirm } from './utils/output.js';
 
 export function registerEvalCommands(program: Command): void {
@@ -242,6 +243,47 @@ export function registerEvalCommands(program: Command): void {
       }
     });
 
+  eval_
+    .command('export')
+    .argument('<run-id>', 'Evaluation run ID')
+    .description('Export evaluation results as markdown')
+    .option('--output <path>', 'Output file path')
+    .action(async (runId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const spinner = ora('Fetching evaluation data...').start();
+
+        const result = await exportEvalMarkdown(runId, options.output, client);
+
+        spinner.stop();
+        printSuccess(`Exported to ${result.file_path}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
+  eval_
+    .command('report')
+    .argument('<run-id>', 'Evaluation run ID')
+    .description('Generate branded evaluation report PDF')
+    .option('--output <path>', 'Output file path')
+    .action(async (runId: string, options) => {
+      try {
+        const client = new IndemnClient();
+        const spinner = ora('Fetching evaluation data...').start();
+
+        spinner.text = 'Generating PDF...';
+        const result = await exportEvalReport(runId, options.output, client);
+
+        spinner.stop();
+        printSuccess(`Report saved to ${result.file_path}`);
+      } catch (err) {
+        printError(err);
+        process.exit(1);
+      }
+    });
+
   // Rubric commands — registered on program, not nested under eval
   const rubric = program.command('rubric').description('Manage evaluation rubrics');
 

package/src/cli/index.ts

@@ -13,7 +13,7 @@ const program = new Command();
 program
   .name('indemn')
   .description('CLI for managing Indemn AI agents')
-  .version('1.0.0');
+  .version('1.1.1');
 
 program.option('--env <environment>', 'Environment (dev or production)');
 

package/src/mcp/tools.ts

@@ -9,6 +9,7 @@ import { RubricsSDK } from '../sdk/rubrics.js';
 import { TestSetsSDK } from '../sdk/test-sets.js';
 import { EvalSDK } from '../sdk/eval.js';
 import { ChatSession } from '../sdk/chat.js';
+import { exportEvalMarkdown, exportEvalReport, exportAgentCard } from '../report/export.js';
 
 // Helper: format a successful MCP tool result
 function success(data: unknown) {
@@ -968,4 +969,53 @@ export function registerTools(server: McpServer, client: IndemnClient): void {
       } catch (err) { return error(err); }
     },
   );
+
+  // ---------------------------------------------------------------------------
+  // Export tools
+  // ---------------------------------------------------------------------------
+
+  server.tool(
+    'export_eval_markdown',
+    'Export evaluation results as a structured markdown file with conversations, scores, and criteria',
+    {
+      run_id: z.string().describe('The evaluation run ID'),
+      output_path: z.string().optional().describe('Output file path (defaults to current directory)'),
+    },
+    async (args) => {
+      try {
+        const result = await exportEvalMarkdown(args.run_id, args.output_path, client);
+        return success(result);
+      } catch (err) { return error(err); }
+    },
+  );
+
+  server.tool(
+    'export_eval_report',
+    'Generate a branded PDF evaluation report with cover page, scores, and per-item details',
+    {
+      run_id: z.string().describe('The evaluation run ID'),
+      output_path: z.string().optional().describe('Output file path (defaults to current directory)'),
+    },
+    async (args) => {
+      try {
+        const result = await exportEvalReport(args.run_id, args.output_path, client);
+        return success(result);
+      } catch (err) { return error(err); }
+    },
+  );
+
+  server.tool(
+    'export_agent_card',
+    'Generate a branded PDF agent card with system prompt, tools, KBs, LLM config, and evaluation history',
+    {
+      agent_id: z.string().describe('The agent ID'),
+      output_path: z.string().optional().describe('Output file path (defaults to current directory)'),
+    },
+    async (args) => {
+      try {
+        const result = await exportAgentCard(args.agent_id, args.output_path, client);
+        return success(result);
+      } catch (err) { return error(err); }
+    },
+  );
 }