agentv 2.6.0 → 2.7.1-next.3

package/dist/templates/.claude/skills/agentv-eval-builder/references/custom-evaluators.md

@@ -7,7 +7,7 @@
 ```json
 {
   "question": "string",
-  "expected_outcome": "string",
+  "criteria": "string",
   "reference_answer": "string",
   "candidate_answer": "string",
   "guideline_files": ["path"],
@@ -20,9 +20,12 @@
     "tool_names": ["fetch"],
     "tool_calls_by_name": {"fetch": 1},
     "error_count": 0,
+    "llm_call_count": 2,
     "token_usage": {"input": 1000, "output": 500},
     "cost_usd": 0.0015,
-    "duration_ms": 3500
+    "duration_ms": 3500,
+    "start_time": "2026-02-13T10:00:00.000Z",
+    "end_time": "2026-02-13T10:00:03.500Z"
   }
 }
 ```
@@ -51,7 +54,7 @@ import { defineCodeJudge, createTargetClient, definePromptTemplate } from '@agen
   - `.invoke({question, systemPrompt})` - Single LLM call
   - `.invokeBatch(requests)` - Batch LLM calls
 - `definePromptTemplate(fn)` - Wraps prompt generation function
-  - Context fields: `question`, `candidateAnswer`, `referenceAnswer`, `expectedOutcome`, `expectedMessages`, `outputMessages`, `config`, `traceSummary`
+  - Context fields: `question`, `candidateAnswer`, `referenceAnswer`, `criteria`, `expectedMessages`, `outputMessages`, `config`, `traceSummary`
 
 ## Python Example
 
@@ -83,10 +86,10 @@ if __name__ == "__main__":
 #!/usr/bin/env bun
 import { defineCodeJudge } from '@agentv/eval';
 
-export default defineCodeJudge(({ candidateAnswer, expectedOutcome }) => {
+export default defineCodeJudge(({ candidateAnswer, criteria }) => {
   const hits: string[] = [];
   const misses: string[] = [];
-  if (candidateAnswer.includes(expectedOutcome)) {
+  if (candidateAnswer.includes(criteria)) {
     hits.push('Matches expected outcome');
   } else {
     misses.push('Does not match expected outcome');
@@ -105,7 +108,7 @@ Derived from eval case fields (users never author these directly):
 | Variable | Source |
 |----------|--------|
 | `question` | First user message in `input_messages` |
-| `expected_outcome` | Eval case `expected_outcome` field |
+| `criteria` | Eval case `criteria` field |
 | `reference_answer` | Last entry in `expected_messages` |
 | `candidate_answer` | Last entry in `output_messages` (runtime) |
 | `input_messages` | Full resolved input array (JSON) |

package/dist/templates/.claude/skills/agentv-eval-builder/references/eval-schema.json

@@ -60,7 +60,7 @@
       },
       "additionalProperties": true
     },
-    "evalcases": {
+    "cases": {
       "type": "array",
       "description": "Array of evaluation cases",
       "minItems": 1,
@@ -75,7 +75,7 @@
             "type": "string",
             "description": "Optional conversation identifier for threading multiple eval cases together"
           },
-          "expected_outcome": {
+          "criteria": {
             "type": "string",
             "description": "Description of what the AI should accomplish in this eval"
           },
@@ -267,12 +267,12 @@
             "additionalProperties": true
           }
         },
-        "required": ["id", "expected_outcome"],
+        "required": ["id", "criteria"],
         "anyOf": [{ "required": ["input_messages"] }, { "required": ["input"] }],
         "additionalProperties": true
       }
     }
   },
-  "required": ["evalcases"],
+  "required": ["cases"],
   "additionalProperties": false
 }

package/dist/templates/.claude/skills/agentv-eval-builder/references/rubric-evaluator.md

@@ -5,25 +5,23 @@
 | Field | Type | Default | Description |
 |-------|------|---------|-------------|
 | `id` | string | auto-generated | Unique identifier |
-| `expected_outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) |
+| `outcome` | string | required* | Criterion being evaluated (*optional if `score_ranges` used) |
 | `weight` | number | 1.0 | Relative importance |
 | `required` | boolean | true | Failing forces verdict to 'fail' (checklist mode) |
 | `required_min_score` | integer | - | Minimum 0-10 score to pass (score-range mode) |
 | `score_ranges` | map or array | - | Score range definitions for analytic scoring |
 
-`description` is a backward-compatible alias for `expected_outcome`.
-
 ## Checklist Mode
 
 ```yaml
 rubrics:
   - Mentions divide-and-conquer approach
   - id: complexity
-    expected_outcome: States time complexity correctly
+    outcome: States time complexity correctly
     weight: 2.0
     required: true
   - id: examples
-    expected_outcome: Includes code examples
+    outcome: Includes code examples
     weight: 1.0
     required: false
 ```
@@ -51,13 +49,13 @@ Array format is also accepted:
 ```yaml
     score_ranges:
       - score_range: [0, 2]
-        expected_outcome: Critical bugs
+        outcome: Critical bugs
       - score_range: [3, 5]
-        expected_outcome: Minor bugs
+        outcome: Minor bugs
       - score_range: [6, 8]
-        expected_outcome: Correct with minor issues
+        outcome: Correct with minor issues
       - score_range: [9, 10]
-        expected_outcome: Fully correct
+        outcome: Fully correct
 ```
 
 Ranges must be integers 0-10, non-overlapping, covering all values 0-10.

package/dist/templates/.github/prompts/agentv-eval-build.prompt.md

@@ -1,5 +1,5 @@
----
-description: 'Create and maintain AgentV YAML evaluation files'
----
-
+---
+description: 'Create and maintain AgentV YAML evaluation files'
+---
+
 #file:../../.claude/skills/agentv-eval-builder/SKILL.md

package/dist/templates/.github/prompts/agentv-optimize.prompt.md

@@ -1,4 +1,4 @@
----
-description: Iteratively optimize prompt files against an AgentV evaluation suite
----
+---
+description: Iteratively optimize prompt files against an AgentV evaluation suite
+---
 #file:../../.claude/skills/agentv-prompt-optimizer/SKILL.md

package/dist/{token-D3IYDJQZ.js → token-POXF46NU.js}

@@ -1,11 +1,13 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   require_token_util
-} from "./chunk-LJVS3JAK.js";
+} from "./chunk-BL4PVUAT.js";
 import {
-  __commonJS,
   require_token_error
-} from "./chunk-BKMQNEUD.js";
+} from "./chunk-3L2L5GIL.js";
+import {
+  __commonJS
+} from "./chunk-5H446C7X.js";
 
 // ../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token.js
 var require_token = __commonJS({
@@ -61,4 +63,4 @@ var require_token = __commonJS({
   }
 });
 export default require_token();
-//# sourceMappingURL=token-D3IYDJQZ.js.map
+//# sourceMappingURL=token-POXF46NU.js.map

package/dist/{token-D3IYDJQZ.js.map → token-POXF46NU.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token.js"],"sourcesContent":["\"use strict\";\nvar __defProp = Object.defineProperty;\nvar __getOwnPropDesc = Object.getOwnPropertyDescriptor;\nvar __getOwnPropNames = Object.getOwnPropertyNames;\nvar __hasOwnProp = Object.prototype.hasOwnProperty;\nvar __export = (target, all) => {\n  for (var name in all)\n    __defProp(target, name, { get: all[name], enumerable: true });\n};\nvar __copyProps = (to, from, except, desc) => {\n  if (from && typeof from === \"object\" || typeof from === \"function\") {\n    for (let key of __getOwnPropNames(from))\n      if (!__hasOwnProp.call(to, key) && key !== except)\n        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });\n  }\n  return to;\n};\nvar __toCommonJS = (mod) => __copyProps(__defProp({}, \"__esModule\", { value: true }), mod);\nvar token_exports = {};\n__export(token_exports, {\n  refreshToken: () => refreshToken\n});\nmodule.exports = __toCommonJS(token_exports);\nvar import_token_error = require(\"./token-error\");\nvar import_token_util = require(\"./token-util\");\nasync function refreshToken() {\n  const { projectId, teamId } = (0, import_token_util.findProjectInfo)();\n  let maybeToken = (0, import_token_util.loadToken)(projectId);\n  if (!maybeToken || (0, import_token_util.isExpired)((0, import_token_util.getTokenPayload)(maybeToken.token))) {\n    const authToken = (0, import_token_util.getVercelCliToken)();\n    if (!authToken) {\n      throw new import_token_error.VercelOidcTokenError(\n        \"Failed to refresh OIDC token: login to vercel cli\"\n      );\n    }\n    if (!projectId) {\n      throw new import_token_error.VercelOidcTokenError(\n        \"Failed to refresh OIDC token: project id not found\"\n      );\n    }\n    maybeToken = await (0, import_token_util.getVercelOidcToken)(authToken, projectId, teamId);\n    if (!maybeToken) {\n      throw new import_token_error.VercelOidcTokenError(\"Failed to refresh OIDC token\");\n    }\n    (0, import_token_util.saveToken)(maybeToken, projectId);\n  }\n  process.env.VERCEL_OIDC_TOKEN = maybeToken.token;\n  return;\n}\n// Annotate the CommonJS export names for ESM import in node:\n0 && (module.exports = {\n  refreshToken\n});\n"],"mappings":";;;;;;;;;;AAAA;AAAA;AACA,QAAI,YAAY,OAAO;AACvB,QAAI,mBAAmB,OAAO;AAC9B,QAAI,oBAAoB,OAAO;AAC/B,QAAI,eAAe,OAAO,UAAU;AACpC,QAAI,WAAW,CAAC,QAAQ,QAAQ;AAC9B,eAAS,QAAQ;AACf,kBAAU,QAAQ,MAAM,EAAE,KAAK,IAAI,IAAI,GAAG,YAAY,KAAK,CAAC;AAAA,IAChE;AACA,QAAI,cAAc,CAAC,IAAI,MAAM,QAAQ,SAAS;AAC5C,UAAI,QAAQ,OAAO,SAAS,YAAY,OAAO,SAAS,YAAY;AAClE,iBAAS,OAAO,kBAAkB,IAAI;AACpC,cAAI,CAAC,aAAa,KAAK,IAAI,GAAG,KAAK,QAAQ;AACzC,sBAAU,IAAI,KAAK,EAAE,KAAK,MAAM,KAAK,GAAG,GAAG,YAAY,EAAE,OAAO,iBAAiB,MAAM,GAAG,MAAM,KAAK,WAAW,CAAC;AAAA,MACvH;AACA,aAAO;AAAA,IACT;AACA,QAAI,eAAe,CAAC,QAAQ,YAAY,UAAU,CAAC,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,GAAG,GAAG;AACzF,QAAI,gBAAgB,CAAC;AACrB,aAAS,eAAe;AAAA,MACtB,cAAc,MAAM;AAAA,IACtB,CAAC;AACD,WAAO,UAAU,aAAa,aAAa;AAC3C,QAAI,qBAAqB;AACzB,QAAI,oBAAoB;AACxB,mBAAe,eAAe;AAC5B,YAAM,EAAE,WAAW,OAAO,KAAK,GAAG,kBAAkB,iBAAiB;AACrE,UAAI,cAAc,GAAG,kBAAkB,WAAW,SAAS;AAC3D,UAAI,CAAC,eAAe,GAAG,kBAAkB,YAAY,GAAG,kBAAkB,iBAAiB,WAAW,KAAK,CAAC,GAAG;AAC7G,cAAM,aAAa,GAAG,kBAAkB,mBAAmB;AAC3D,YAAI,CAAC,WAAW;AACd,gBAAM,IAAI,mBAAmB;AAAA,YAC3B;AAAA,UACF;AAAA,QACF;AACA,YAAI,CAAC,WAAW;AACd,gBAAM,IAAI,mBAAmB;AAAA,YAC3B;AAAA,UACF;AAAA,QACF;AACA,qBAAa,OAAO,GAAG,kBAAkB,oBAAoB,WAAW,WAAW,MAAM;AACzF,YAAI,CAAC,YAAY;AACf,gBAAM,IAAI,mBAAmB,qBAAqB,8BAA8B;AAAA,QAClF;AACA,SAAC,GAAG,kBAAkB,WAAW,YAAY,SAAS;AAAA,MACxD;AACA,cAAQ,IAAI,oBAAoB,WAAW;AAC3C;AAAA,IACF;AAAA;AAAA;","names":[]}
+{"version":3,"sources":["../../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token.js"],"sourcesContent":["\"use strict\";\nvar __defProp = Object.defineProperty;\nvar __getOwnPropDesc = Object.getOwnPropertyDescriptor;\nvar __getOwnPropNames = Object.getOwnPropertyNames;\nvar __hasOwnProp = Object.prototype.hasOwnProperty;\nvar __export = (target, all) => {\n  for (var name in all)\n    __defProp(target, name, { get: all[name], enumerable: true });\n};\nvar __copyProps = (to, from, except, desc) => {\n  if (from && typeof from === \"object\" || typeof from === \"function\") {\n    for (let key of __getOwnPropNames(from))\n      if (!__hasOwnProp.call(to, key) && key !== except)\n        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });\n  }\n  return to;\n};\nvar __toCommonJS = (mod) => __copyProps(__defProp({}, \"__esModule\", { value: true }), mod);\nvar token_exports = {};\n__export(token_exports, {\n  refreshToken: () => refreshToken\n});\nmodule.exports = __toCommonJS(token_exports);\nvar import_token_error = require(\"./token-error\");\nvar import_token_util = require(\"./token-util\");\nasync function refreshToken() {\n  const { projectId, teamId } = (0, import_token_util.findProjectInfo)();\n  let maybeToken = (0, import_token_util.loadToken)(projectId);\n  if (!maybeToken || (0, import_token_util.isExpired)((0, import_token_util.getTokenPayload)(maybeToken.token))) {\n    const authToken = (0, import_token_util.getVercelCliToken)();\n    if (!authToken) {\n      throw new import_token_error.VercelOidcTokenError(\n        \"Failed to refresh OIDC token: login to vercel cli\"\n      );\n    }\n    if (!projectId) {\n      throw new import_token_error.VercelOidcTokenError(\n        \"Failed to refresh OIDC token: project id not found\"\n      );\n    }\n    maybeToken = await (0, import_token_util.getVercelOidcToken)(authToken, projectId, teamId);\n    if (!maybeToken) {\n      throw new import_token_error.VercelOidcTokenError(\"Failed to refresh OIDC token\");\n    }\n    (0, import_token_util.saveToken)(maybeToken, projectId);\n  }\n  process.env.VERCEL_OIDC_TOKEN = maybeToken.token;\n  return;\n}\n// Annotate the CommonJS export names for ESM import in node:\n0 && (module.exports = {\n  refreshToken\n});\n"],"mappings":";;;;;;;;;;;;AAAA;AAAA;AACA,QAAI,YAAY,OAAO;AACvB,QAAI,mBAAmB,OAAO;AAC9B,QAAI,oBAAoB,OAAO;AAC/B,QAAI,eAAe,OAAO,UAAU;AACpC,QAAI,WAAW,CAAC,QAAQ,QAAQ;AAC9B,eAAS,QAAQ;AACf,kBAAU,QAAQ,MAAM,EAAE,KAAK,IAAI,IAAI,GAAG,YAAY,KAAK,CAAC;AAAA,IAChE;AACA,QAAI,cAAc,CAAC,IAAI,MAAM,QAAQ,SAAS;AAC5C,UAAI,QAAQ,OAAO,SAAS,YAAY,OAAO,SAAS,YAAY;AAClE,iBAAS,OAAO,kBAAkB,IAAI;AACpC,cAAI,CAAC,aAAa,KAAK,IAAI,GAAG,KAAK,QAAQ;AACzC,sBAAU,IAAI,KAAK,EAAE,KAAK,MAAM,KAAK,GAAG,GAAG,YAAY,EAAE,OAAO,iBAAiB,MAAM,GAAG,MAAM,KAAK,WAAW,CAAC;AAAA,MACvH;AACA,aAAO;AAAA,IACT;AACA,QAAI,eAAe,CAAC,QAAQ,YAAY,UAAU,CAAC,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,GAAG,GAAG;AACzF,QAAI,gBAAgB,CAAC;AACrB,aAAS,eAAe;AAAA,MACtB,cAAc,MAAM;AAAA,IACtB,CAAC;AACD,WAAO,UAAU,aAAa,aAAa;AAC3C,QAAI,qBAAqB;AACzB,QAAI,oBAAoB;AACxB,mBAAe,eAAe;AAC5B,YAAM,EAAE,WAAW,OAAO,KAAK,GAAG,kBAAkB,iBAAiB;AACrE,UAAI,cAAc,GAAG,kBAAkB,WAAW,SAAS;AAC3D,UAAI,CAAC,eAAe,GAAG,kBAAkB,YAAY,GAAG,kBAAkB,iBAAiB,WAAW,KAAK,CAAC,GAAG;AAC7G,cAAM,aAAa,GAAG,kBAAkB,mBAAmB;AAC3D,YAAI,CAAC,WAAW;AACd,gBAAM,IAAI,mBAAmB;AAAA,YAC3B;AAAA,UACF;AAAA,QACF;AACA,YAAI,CAAC,WAAW;AACd,gBAAM,IAAI,mBAAmB;AAAA,YAC3B;AAAA,UACF;AAAA,QACF;AACA,qBAAa,OAAO,GAAG,kBAAkB,oBAAoB,WAAW,WAAW,MAAM;AACzF,YAAI,CAAC,YAAY;AACf,gBAAM,IAAI,mBAAmB,qBAAqB,8BAA8B;AAAA,QAClF;AACA,SAAC,GAAG,kBAAkB,WAAW,YAAY,SAAS;AAAA,MACxD;AACA,cAAQ,IAAI,oBAAoB,WAAW;AAC3C;AAAA,IACF;AAAA;AAAA;","names":[]}

package/dist/{token-util-FWFPR2BV.js → token-util-6GWYZWGE.js}

@@ -1,7 +1,8 @@
 import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
 import {
   require_token_util
-} from "./chunk-LJVS3JAK.js";
-import "./chunk-BKMQNEUD.js";
+} from "./chunk-BL4PVUAT.js";
+import "./chunk-3L2L5GIL.js";
+import "./chunk-5H446C7X.js";
 export default require_token_util();
-//# sourceMappingURL=token-util-FWFPR2BV.js.map
+//# sourceMappingURL=token-util-6GWYZWGE.js.map

package/dist/token-util-6GWYZWGE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "2.6.0",
+  "version": "2.7.1-next.3",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {
@@ -16,7 +16,7 @@
   },
   "files": ["dist", "README.md"],
   "scripts": {
-    "dev": "bun --watch src/index.ts",
+    "dev": "bun src/cli.ts",
     "build": "tsup && bun run copy-readme",
     "copy-readme": "bun -e \"import { cpSync } from 'fs'; cpSync('../../README.md', 'README.md')\"",
     "prepublishOnly": "bun run copy-readme",
@@ -28,9 +28,12 @@
     "test:watch": "bun test --watch"
   },
   "dependencies": {
-    "@agentv/core": "2.5.5",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.49",
+    "@github/copilot-sdk": "^0.1.25",
+    "@inquirer/prompts": "^8.2.1",
     "@mariozechner/pi-agent-core": "^0.50.9",
     "@mariozechner/pi-ai": "^0.50.9",
+    "@openai/codex-sdk": "^0.104.0",
     "cmd-ts": "^0.14.3",
     "dotenv": "^16.4.5",
     "fast-glob": "^3.3.3",
@@ -39,6 +42,7 @@
     "yaml": "^2.6.1"
   },
   "devDependencies": {
+    "@agentv/core": "2.7.0",
     "execa": "^9.3.0"
   }
 }