agentv 3.9.2 → 3.10.1

package/dist/cli.js

@@ -2,9 +2,9 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   runCli
-} from "./chunk-6ZAFWUBT.js";
-import "./chunk-JGMJL2LV.js";
-import "./chunk-OIVGGWJ3.js";
+} from "./chunk-JLFFYTZA.js";
+import "./chunk-TXCVDTEE.js";
+import "./chunk-GWHHM6X2.js";
 import "./chunk-C5GOHBQM.js";
 import "./chunk-JK6V4KVD.js";
 import "./chunk-HQDCIXVH.js";

package/dist/{dist-PUPHGVKL.js → dist-FPC7J7KQ.js}

@@ -141,7 +141,7 @@ import {
   transpileEvalYaml,
   transpileEvalYamlFile,
   trimBaselineResult
-} from "./chunk-OIVGGWJ3.js";
+} from "./chunk-GWHHM6X2.js";
 import {
   OtlpJsonFileExporter
 } from "./chunk-C5GOHBQM.js";
@@ -300,4 +300,4 @@ export {
   transpileEvalYamlFile,
   trimBaselineResult
 };
-//# sourceMappingURL=dist-PUPHGVKL.js.map
+//# sourceMappingURL=dist-FPC7J7KQ.js.map

package/dist/index.js

@@ -3,9 +3,9 @@ import {
   app,
   preprocessArgv,
   runCli
-} from "./chunk-6ZAFWUBT.js";
-import "./chunk-JGMJL2LV.js";
-import "./chunk-OIVGGWJ3.js";
+} from "./chunk-JLFFYTZA.js";
+import "./chunk-TXCVDTEE.js";
+import "./chunk-GWHHM6X2.js";
 import "./chunk-C5GOHBQM.js";
 import "./chunk-JK6V4KVD.js";
 import "./chunk-HQDCIXVH.js";

package/dist/{interactive-BD56NB23.js → interactive-N463HRIL.js}

@@ -4,14 +4,14 @@ import {
   fileExists,
   findRepoRoot,
   runEvalCommand
-} from "./chunk-JGMJL2LV.js";
+} from "./chunk-TXCVDTEE.js";
 import {
   DEFAULT_EVAL_PATTERNS,
   getAgentvHome,
   listTargetNames,
   loadConfig,
   readTargetDefinitions
-} from "./chunk-OIVGGWJ3.js";
+} from "./chunk-GWHHM6X2.js";
 import "./chunk-C5GOHBQM.js";
 import "./chunk-JK6V4KVD.js";
 import "./chunk-HQDCIXVH.js";
@@ -371,4 +371,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
 export {
   launchInteractiveWizard
 };
-//# sourceMappingURL=interactive-BD56NB23.js.map
+//# sourceMappingURL=interactive-N463HRIL.js.map

package/dist/templates/.agents/skills/agentv-chat-to-eval/README.md

@@ -0,0 +1,84 @@
+# agentv-chat-to-eval
+
+An AgentV skill that converts chat conversations into evaluation YAML files.
+
+## What It Does
+
+This skill takes a chat transcript — either as markdown conversation or JSON messages — and generates an AgentV-compatible eval file with test cases derived from the exchanges.
+
+The LLM analyzes the conversation to:
+1. Identify test-worthy exchanges (factual Q&A, task completion, edge cases)
+2. Derive evaluation criteria from context
+3. Generate valid YAML with `tests:`, `assert` evaluators, and rubrics
+
+## Usage
+
+Provide a chat transcript and ask the agent to convert it:
+
+```
+Convert this conversation into an AgentV eval file:
+
+User: What's the capital of France?
+Assistant: The capital of France is Paris.
+
+User: How do I reverse a list in Python?
+Assistant: Use the `reverse()` method or slicing: `my_list[::-1]`
+```
+
+Or provide a JSON message array:
+
+```json
+[
+  {"role": "user", "content": "What's the capital of France?"},
+  {"role": "assistant", "content": "The capital of France is Paris."}
+]
+```
+
+## Example Output
+
+```yaml
+description: "General knowledge and coding Q&A"
+
+tests:
+  - id: capital-of-france
+    criteria: "Correctly identify the capital of France"
+    input: "What's the capital of France?"
+    expected_output: "The capital of France is Paris."
+    assert:
+      - type: rubrics
+        criteria:
+          - States Paris as the capital
+          - Response is concise and direct
+
+  - id: python-reverse-list
+    criteria: "Explain how to reverse a list in Python"
+    input: "How do I reverse a list in Python?"
+    expected_output: "Use the `reverse()` method or slicing: `my_list[::-1]`"
+    assert:
+      - type: rubrics
+        criteria:
+          - Provides at least one valid method to reverse a list
+          - Code syntax is correct
+          - Explanation is clear and actionable
+
+# Suggested additional evaluators:
+# assert:
+#   - name: quality
+#     type: llm_judge
+#     prompt: ./prompts/quality.md
+#   - name: accuracy
+#     type: code_judge
+#     command: [./scripts/check_accuracy.py]
+```
+
+## When to Use
+
+- You have a real conversation that demonstrates desired agent behavior
+- You want to create regression tests from production interactions
+- You're bootstrapping an eval suite from existing chat logs
+- You need to convert Q&A pairs into structured test cases
+
+## Related Skills
+
+- **agentv-eval-builder** — Create eval files from scratch with full schema reference
+- **agentv-eval-orchestrator** — Run evaluations without API keys

package/dist/templates/.agents/skills/agentv-chat-to-eval/SKILL.md

@@ -0,0 +1,144 @@
+---
+name: agentv-chat-to-eval
+description: Convert chat conversations into AgentV evaluation YAML files. Use this skill when you have a chat transcript (markdown or JSON messages) and want to generate eval test cases from it.
+---
+
+# Chat-to-Eval Converter
+
+Convert chat transcripts into AgentV evaluation YAML files by extracting test-worthy exchanges.
+
+## Input Variables
+
+- `transcript`: Chat transcript as markdown conversation or JSON message array
+- `eval-path` (optional): Output path for the generated YAML file
+
+## Workflow
+
+### 1. Parse the Transcript
+
+Accept input in either format:
+
+**Markdown conversation:**
+```
+User: How do I reset my password?
+Assistant: Go to Settings > Security > Reset Password...
+```
+
+**JSON messages:**
+```json
+[
+  {"role": "user", "content": "How do I reset my password?"},
+  {"role": "assistant", "content": "Go to Settings > Security > Reset Password..."}
+]
+```
+
+Normalize both formats into `{role, content}` message pairs.
+
+### 2. Identify Test-Worthy Exchanges
+
+Extract exchanges that are good eval candidates. Prioritize:
+
+- **Factual Q&A** — User asks a question, agent gives a verifiable answer
+- **Task completion** — User requests an action, agent performs it
+- **Multi-turn reasoning** — Exchanges where context from earlier turns matters
+- **Edge cases** — Unusual inputs, error handling, boundary conditions
+- **Domain expertise** — Responses requiring specialized knowledge
+
+Skip:
+- Greetings and small talk (unless testing social behavior)
+- Acknowledgments without substance ("OK", "Got it")
+- Repeated or redundant exchanges
+
+### 3. Derive Criteria and Rubrics
+
+For each selected exchange, infer evaluation criteria from the conversation context:
+
+- What the user implicitly expected
+- Quality signals in the assistant's response (accuracy, completeness, tone)
+- Any corrections or follow-ups that reveal what "good" looks like
+
+Generate rubrics that capture these quality dimensions.
+
+### 4. Generate EVAL YAML
+
+Produce a valid AgentV eval file using **`tests:`** (not `cases:`).
+
+**Structure:**
+
+```yaml
+description: "<Summarize what this eval covers>"
+
+tests:
+  - id: <kebab-case-id>
+    criteria: "<What the response should accomplish>"
+    input: "<User message>"
+    expected_output: "<Assistant response from transcript>"
+    assert:
+      - type: rubrics
+        criteria:
+          - <Quality criterion 1>
+          - <Quality criterion 2>
+```
+
+**Rules:**
+- Use `tests:` as the top-level array key — never `cases:`
+- Generate kebab-case `id` values derived from the exchange topic
+- Write `criteria` as a concise statement of what a good response achieves
+- Use `input` for single user messages; use `input` for multi-turn
+- Set `expected_output` to the actual assistant response from the transcript
+- Include 2–4 rubrics per test as `type: rubrics` under `assert` capturing distinct quality dimensions
+
+### 5. Suggest Evaluators
+
+Append a commented evaluator configuration based on the test content:
+
+```yaml
+# Suggested additional evaluators:
+# assert:
+#   - name: quality
+#     type: llm_judge
+#     prompt: ./prompts/quality.md
+#   - name: accuracy
+#     type: code_judge
+#     command: [./scripts/check_accuracy.py]
+```
+
+- Recommend `llm_judge` for subjective quality (tone, helpfulness, completeness)
+- Recommend `code_judge` for deterministic checks (format, required fields, exact values)
+- Recommend `field_accuracy` when expected output has structured fields
+
+### 6. Write Output
+
+- If `eval-path` is provided, write the YAML to that path
+- Otherwise, output the YAML to the conversation for the user to copy
+
+## Multi-Turn Conversations
+
+For conversations with context dependencies across turns, use `input`:
+
+```yaml
+tests:
+  - id: multi-turn-context
+    criteria: "Agent remembers prior context"
+    input:
+      - role: user
+        content: "My name is Alice"
+      - role: assistant
+        content: "Nice to meet you, Alice!"
+      - role: user
+        content: "What's my name?"
+    expected_output: "Your name is Alice."
+    assert:
+      - type: rubrics
+        criteria:
+          - Correctly recalls the user's name from earlier in the conversation
+          - Response is natural and conversational
+```
+
+## Guidelines
+
+- **Preserve original wording** in `expected_output` — use the actual transcript text
+- **Be selective** — not every exchange makes a good test; aim for 5–15 tests per transcript
+- **Diverse coverage** — pick exchanges that test different capabilities
+- **Actionable rubrics** — each rubric should be independently evaluable (pass/fail)
+- **Validate output** — the generated YAML must pass `agentv validate <file>`

package/dist/templates/.agents/skills/agentv-chat-to-eval/examples/transcript-json.md

@@ -0,0 +1,67 @@
+# Example: JSON Messages Input
+
+## Input
+
+```json
+[
+  {"role": "user", "content": "Convert 72°F to Celsius"},
+  {"role": "assistant", "content": "72°F is approximately 22.2°C. The formula is: (°F - 32) × 5/9 = °C"},
+  {"role": "user", "content": "What about -40?"},
+  {"role": "assistant", "content": "-40°F equals exactly -40°C. This is the unique point where Fahrenheit and Celsius scales intersect."},
+  {"role": "user", "content": "Is 0 Kelvin the same as -273.15°C?"},
+  {"role": "assistant", "content": "Yes, 0 Kelvin (absolute zero) equals -273.15°C. It's the theoretical lowest possible temperature where all molecular motion stops."}
+]
+```
+
+## Output
+
+```yaml
+description: "Temperature conversion and thermodynamics knowledge"
+
+tests:
+  - id: fahrenheit-to-celsius
+    criteria: "Convert Fahrenheit to Celsius with correct formula"
+    input: "Convert 72°F to Celsius"
+    expected_output: "72°F is approximately 22.2°C. The formula is: (°F - 32) × 5/9 = °C"
+    assert:
+      - type: rubrics
+        criteria:
+          - Provides the correct converted value (approximately 22.2°C)
+          - Shows or references the conversion formula
+          - Uses appropriate precision
+
+  - id: negative-forty-intersection
+    criteria: "Identify the Fahrenheit-Celsius intersection point"
+    input:
+      - role: user
+        content: "Convert 72°F to Celsius"
+      - role: assistant
+        content: "72°F is approximately 22.2°C."
+      - role: user
+        content: "What about -40?"
+    expected_output: "-40°F equals exactly -40°C. This is the unique point where Fahrenheit and Celsius scales intersect."
+    assert:
+      - type: rubrics
+        criteria:
+          - States that -40°F equals -40°C
+          - Mentions this is the intersection point of both scales
+
+  - id: absolute-zero
+    criteria: "Confirm absolute zero equivalence and explain its significance"
+    input: "Is 0 Kelvin the same as -273.15°C?"
+    expected_output: "Yes, 0 Kelvin (absolute zero) equals -273.15°C. It's the theoretical lowest possible temperature where all molecular motion stops."
+    assert:
+      - type: rubrics
+        criteria:
+          - Confirms the equivalence of 0K and -273.15°C
+          - Explains the physical significance of absolute zero
+
+# Suggested additional evaluators:
+# assert:
+#   - name: accuracy
+#     type: code_judge
+#     command: [./scripts/check_conversion.py]
+#   - name: quality
+#     type: llm_judge
+#     prompt: ./prompts/quality.md
+```

package/dist/templates/.agents/skills/agentv-chat-to-eval/examples/transcript-markdown.md

@@ -0,0 +1,101 @@
+# Example: Markdown Transcript Input
+
+## Input
+
+```
+User: What are the SOLID principles in software engineering?
+Assistant: The SOLID principles are five design principles for object-oriented programming:
+
+1. **Single Responsibility** — A class should have only one reason to change
+2. **Open/Closed** — Open for extension, closed for modification
+3. **Liskov Substitution** — Subtypes must be substitutable for their base types
+4. **Interface Segregation** — Prefer small, specific interfaces over large ones
+5. **Dependency Inversion** — Depend on abstractions, not concretions
+
+User: Can you give me an example of the Single Responsibility Principle?
+Assistant: Sure! Instead of a class that handles both user authentication and email sending, split them:
+
+Bad: `UserService` handles login, registration, AND sends welcome emails.
+Good: `AuthService` handles login/registration. `EmailService` handles sending emails.
+
+Each class has one job, making them easier to test and maintain.
+
+User: What's the difference between an interface and an abstract class?
+Assistant: Key differences:
+
+- **Interface**: Defines a contract (method signatures only). A class can implement multiple interfaces. No state.
+- **Abstract class**: Can have both abstract and concrete methods. A class can extend only one. Can hold state.
+
+Use interfaces for "can-do" relationships (e.g., `Serializable`). Use abstract classes for "is-a" relationships with shared behavior (e.g., `Animal` base class).
+```
+
+## Output
+
+```yaml
+description: "Software engineering concepts — SOLID principles and OOP"
+
+tests:
+  - id: solid-principles-overview
+    criteria: "List and briefly explain all five SOLID principles"
+    input: "What are the SOLID principles in software engineering?"
+    expected_output: |
+      The SOLID principles are five design principles for object-oriented programming:
+
+      1. **Single Responsibility** — A class should have only one reason to change
+      2. **Open/Closed** — Open for extension, closed for modification
+      3. **Liskov Substitution** — Subtypes must be substitutable for their base types
+      4. **Interface Segregation** — Prefer small, specific interfaces over large ones
+      5. **Dependency Inversion** — Depend on abstractions, not concretions
+    assert:
+      - type: rubrics
+        criteria:
+          - Lists all five SOLID principles by name
+          - Provides a concise explanation for each principle
+          - Explanations are technically accurate
+
+  - id: srp-example
+    criteria: "Provide a concrete example of the Single Responsibility Principle"
+    input:
+      - role: user
+        content: "What are the SOLID principles in software engineering?"
+      - role: assistant
+        content: "The SOLID principles are five design principles..."
+      - role: user
+        content: "Can you give me an example of the Single Responsibility Principle?"
+    expected_output: |
+      Sure! Instead of a class that handles both user authentication and email sending, split them:
+
+      Bad: `UserService` handles login, registration, AND sends welcome emails.
+      Good: `AuthService` handles login/registration. `EmailService` handles sending emails.
+
+      Each class has one job, making them easier to test and maintain.
+    assert:
+      - type: rubrics
+        criteria:
+          - Shows a bad example that violates SRP
+          - Shows a good example that follows SRP
+          - Explains why the separation is beneficial
+
+  - id: interface-vs-abstract-class
+    criteria: "Explain the difference between interfaces and abstract classes"
+    input: "What's the difference between an interface and an abstract class?"
+    expected_output: |
+      Key differences:
+
+      - **Interface**: Defines a contract (method signatures only). A class can implement multiple interfaces. No state.
+      - **Abstract class**: Can have both abstract and concrete methods. A class can extend only one. Can hold state.
+
+      Use interfaces for "can-do" relationships (e.g., `Serializable`). Use abstract classes for "is-a" relationships with shared behavior (e.g., `Animal` base class).
+    assert:
+      - type: rubrics
+        criteria:
+          - Correctly distinguishes interfaces from abstract classes
+          - Mentions multiple inheritance support for interfaces
+          - Provides guidance on when to use each
+
+# Suggested additional evaluators:
+# assert:
+#   - name: quality
+#     type: llm_judge
+#     prompt: ./prompts/quality.md
+```