agentv 2.5.8 → 2.7.1-next.1

package/README.md

@@ -27,19 +27,18 @@ description: Math problem solving evaluation
 execution:
   target: default
 
-evalcases:
+tests:
   - id: addition
-    expected_outcome: Correctly calculates 15 + 27 = 42
+    criteria: Correctly calculates 15 + 27 = 42
 
     input: What is 15 + 27?
 
     expected_output: "42"
 
-    execution:
-      evaluators:
-        - name: math_check
-          type: code_judge
-          script: ./validators/check_math.py
+    assert:
+      - name: math_check
+        type: code_judge
+        script: ./validators/check_math.py
 ```
 
 **5. Run the eval:**
@@ -95,6 +94,25 @@ bun test
 
 See [AGENTS.md](AGENTS.md) for development guidelines and design principles.
 
+### Releasing
+
+Stable release:
+
+```bash
+bun run release          # patch bump
+bun run release minor
+bun run release major
+bun run publish          # publish to npm `latest`
+```
+
+Prerelease (`next`) channel:
+
+```bash
+bun run release:next         # bump/increment `-next.N`
+bun run release:next major   # start new major prerelease line
+bun run publish:next         # publish to npm `next`
+```
+
 ## Core Concepts
 
 **Evaluation files** (`.yaml` or `.jsonl`) define test cases with expected outcomes. **Targets** specify which agent/provider to evaluate. **Judges** (code or LLM) score results. **Results** are written as JSONL/YAML for analysis and comparison.
@@ -104,11 +122,11 @@ See [AGENTS.md](AGENTS.md) for development guidelines and design principles.
 For large-scale evaluations, AgentV supports JSONL (JSON Lines) format as an alternative to YAML:
 
 ```jsonl
-{"id": "test-1", "expected_outcome": "Calculates correctly", "input": "What is 2+2?"}
-{"id": "test-2", "expected_outcome": "Provides explanation", "input": "Explain variables"}
+{"id": "test-1", "criteria": "Calculates correctly", "input": "What is 2+2?"}
+{"id": "test-2", "criteria": "Provides explanation", "input": "Explain variables"}
 ```
 
-Optional sidecar YAML metadata file (`dataset.yaml` alongside `dataset.jsonl`):
+Optional sidecar YAML metadata file (`dataset.eval.yaml` alongside `dataset.jsonl`):
 ```yaml
 description: Math evaluation dataset
 dataset: math-tests
@@ -133,8 +151,8 @@ agentv eval evals/my-eval.yaml
 # Override target
 agentv eval --target azure_base evals/**/*.yaml
 
-# Run specific eval case
-agentv eval --eval-id case-123 evals/my-eval.yaml
+# Run specific test
+agentv eval --test-id case-123 evals/my-eval.yaml
 
 # Dry-run with mock provider
 agentv eval --dry-run evals/my-eval.yaml
@@ -150,12 +168,12 @@ Write code judges in Python or TypeScript:
 # validators/check_answer.py
 import json, sys
 data = json.load(sys.stdin)
-candidate_answer = data.get("candidate_answer", "")
+answer = data.get("answer", "")
 
 hits = []
 misses = []
 
-if "42" in candidate_answer:
+if "42" in answer:
     hits.append("Answer contains correct value (42)")
 else:
     misses.append("Answer does not contain expected value (42)")
@@ -173,15 +191,91 @@ print(json.dumps({
 Reference evaluators in your eval file:
 
 ```yaml
-execution:
-  evaluators:
-    - name: my_validator
-      type: code_judge
-      script: ./validators/check_answer.py
+assert:
+  - name: my_validator
+    type: code_judge
+    script: ./validators/check_answer.py
 ```
 
 For complete templates, examples, and evaluator patterns, see: [custom-evaluators](https://agentv.dev/evaluators/custom-evaluators/)
 
+### TypeScript SDK
+
+#### Custom Assertions with `defineAssertion()`
+
+Create custom assertion types in TypeScript using `@agentv/eval`:
+
+```typescript
+// .agentv/assertions/word-count.ts
+import { defineAssertion } from '@agentv/eval';
+
+export default defineAssertion(({ answer }) => {
+  const wordCount = answer.trim().split(/\s+/).length;
+  return {
+    pass: wordCount >= 3,
+    reasoning: `Output has ${wordCount} words`,
+  };
+});
+```
+
+Files in `.agentv/assertions/` are auto-discovered by filename — use directly in YAML:
+
+```yaml
+assert:
+  - type: word-count    # matches word-count.ts
+  - type: contains
+    value: "Hello"
+```
+
+See the [sdk-custom-assertion example](examples/features/sdk-custom-assertion).
+
+#### Programmatic API with `evaluate()`
+
+Use AgentV as a library — no YAML needed:
+
+```typescript
+import { evaluate } from '@agentv/core';
+
+const { results, summary } = await evaluate({
+  tests: [
+    {
+      id: 'greeting',
+      input: 'Say hello',
+      assert: [{ type: 'contains', value: 'Hello' }],
+    },
+  ],
+});
+
+console.log(`${summary.passed}/${summary.total} passed`);
+```
+
+Auto-discovers `default` target from `.agentv/targets.yaml` and `.env` credentials. See the [sdk-programmatic-api example](examples/features/sdk-programmatic-api).
+
+#### Typed Configuration with `defineConfig()`
+
+Create `agentv.config.ts` at your project root for typed, validated configuration:
+
+```typescript
+import { defineConfig } from '@agentv/core';
+
+export default defineConfig({
+  execution: { workers: 5, maxRetries: 2 },
+  output: { format: 'jsonl', dir: './results' },
+  limits: { maxCostUsd: 10.0 },
+});
+```
+
+See the [sdk-config-file example](examples/features/sdk-config-file).
+
+#### Scaffold Commands
+
+Bootstrap new assertions and eval files:
+
+```bash
+agentv create assertion sentiment   # → .agentv/assertions/sentiment.ts
+agentv create eval my-eval          # → evals/my-eval.eval.yaml + .cases.jsonl
+```
+
 ### Compare Evaluation Results
 
 Run two evaluations and compare them:
@@ -218,7 +312,7 @@ targets:
     judge_target: azure_base
 ```
 
-Supports: `azure`, `anthropic`, `gemini`, `codex`, `pi-coding-agent`, `claude-code`, `vscode`, `vscode-insiders`, `cli`, and `mock`.
+Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`.
 
 Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields.
 
@@ -229,7 +323,7 @@ Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/ta
 Write validators in any language (Python, TypeScript, Node, etc.):
 
 ```bash
-# Input: stdin JSON with question, expected_outcome, candidate_answer
+# Input: stdin JSON with question, criteria, answer
 # Output: stdout JSON with score (0-1), hits, misses, reasoning
 ```
 
@@ -242,30 +336,31 @@ For complete examples and patterns, see:
 Create markdown judge files with evaluation criteria and scoring guidelines:
 
 ```yaml
-execution:
-  evaluators:
-    - name: semantic_check
-      type: llm_judge
-      prompt: ./judges/correctness.md
+assert:
+  - name: semantic_check
+    type: llm_judge
+    prompt: ./judges/correctness.md
 ```
 
 Your judge prompt file defines criteria and scoring guidelines.
 
 ### Rubric-Based Evaluation
 
-Define structured criteria directly in your eval case:
+Define structured criteria directly in your test:
 
 ```yaml
-evalcases:
+tests:
   - id: quicksort-explain
-    expected_outcome: Explain how quicksort works
+    criteria: Explain how quicksort works
 
     input: Explain quicksort algorithm
 
-    rubrics:
-      - Mentions divide-and-conquer approach
-      - Explains partition step
-      - States time complexity
+    assert:
+      - type: rubrics
+        criteria:
+          - Mentions divide-and-conquer approach
+          - Explains partition step
+          - States time complexity
 ```
 
 Scoring: `(satisfied weights) / (total weights)` → verdicts: `pass` (≥0.8), `borderline` (≥0.6), `fail`

package/dist/chunk-3L2L5GIL.js

@@ -0,0 +1,51 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+import {
+  __commonJS
+} from "./chunk-5H446C7X.js";
+
+// ../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js
+var require_token_error = __commonJS({
+  "../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js"(exports, module) {
+    "use strict";
+    var __defProp = Object.defineProperty;
+    var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+    var __getOwnPropNames = Object.getOwnPropertyNames;
+    var __hasOwnProp = Object.prototype.hasOwnProperty;
+    var __export = (target, all) => {
+      for (var name in all)
+        __defProp(target, name, { get: all[name], enumerable: true });
+    };
+    var __copyProps = (to, from, except, desc) => {
+      if (from && typeof from === "object" || typeof from === "function") {
+        for (let key of __getOwnPropNames(from))
+          if (!__hasOwnProp.call(to, key) && key !== except)
+            __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+      }
+      return to;
+    };
+    var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+    var token_error_exports = {};
+    __export(token_error_exports, {
+      VercelOidcTokenError: () => VercelOidcTokenError
+    });
+    module.exports = __toCommonJS(token_error_exports);
+    var VercelOidcTokenError = class extends Error {
+      constructor(message, cause) {
+        super(message);
+        this.name = "VercelOidcTokenError";
+        this.cause = cause;
+      }
+      toString() {
+        if (this.cause) {
+          return `${this.name}: ${this.message}: ${this.cause}`;
+        }
+        return `${this.name}: ${this.message}`;
+      }
+    };
+  }
+});
+
+export {
+  require_token_error
+};
+//# sourceMappingURL=chunk-3L2L5GIL.js.map

package/dist/{chunk-BKMQNEUD.js.map → chunk-3L2L5GIL.js.map}

@@ -1 +1 @@
-{"version":3,"sources":["../../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js"],"sourcesContent":["\"use strict\";\nvar __defProp = Object.defineProperty;\nvar __getOwnPropDesc = Object.getOwnPropertyDescriptor;\nvar __getOwnPropNames = Object.getOwnPropertyNames;\nvar __hasOwnProp = Object.prototype.hasOwnProperty;\nvar __export = (target, all) => {\n  for (var name in all)\n    __defProp(target, name, { get: all[name], enumerable: true });\n};\nvar __copyProps = (to, from, except, desc) => {\n  if (from && typeof from === \"object\" || typeof from === \"function\") {\n    for (let key of __getOwnPropNames(from))\n      if (!__hasOwnProp.call(to, key) && key !== except)\n        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });\n  }\n  return to;\n};\nvar __toCommonJS = (mod) => __copyProps(__defProp({}, \"__esModule\", { value: true }), mod);\nvar token_error_exports = {};\n__export(token_error_exports, {\n  VercelOidcTokenError: () => VercelOidcTokenError\n});\nmodule.exports = __toCommonJS(token_error_exports);\nclass VercelOidcTokenError extends Error {\n  constructor(message, cause) {\n    super(message);\n    this.name = \"VercelOidcTokenError\";\n    this.cause = cause;\n  }\n  toString() {\n    if (this.cause) {\n      return `${this.name}: ${this.message}: ${this.cause}`;\n    }\n    return `${this.name}: ${this.message}`;\n  }\n}\n// Annotate the CommonJS export names for ESM import in node:\n0 && (module.exports = {\n  VercelOidcTokenError\n});\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AACA,QAAIA,aAAY,OAAO;AACvB,QAAIC,oBAAmB,OAAO;AAC9B,QAAIC,qBAAoB,OAAO;AAC/B,QAAIC,gBAAe,OAAO,UAAU;AACpC,QAAIC,YAAW,CAAC,QAAQ,QAAQ;AAC9B,eAAS,QAAQ;AACf,QAAAJ,WAAU,QAAQ,MAAM,EAAE,KAAK,IAAI,IAAI,GAAG,YAAY,KAAK,CAAC;AAAA,IAChE;AACA,QAAIK,eAAc,CAAC,IAAI,MAAM,QAAQ,SAAS;AAC5C,UAAI,QAAQ,OAAO,SAAS,YAAY,OAAO,SAAS,YAAY;AAClE,iBAAS,OAAOH,mBAAkB,IAAI;AACpC,cAAI,CAACC,cAAa,KAAK,IAAI,GAAG,KAAK,QAAQ;AACzC,YAAAH,WAAU,IAAI,KAAK,EAAE,KAAK,MAAM,KAAK,GAAG,GAAG,YAAY,EAAE,OAAOC,kBAAiB,MAAM,GAAG,MAAM,KAAK,WAAW,CAAC;AAAA,MACvH;AACA,aAAO;AAAA,IACT;AACA,QAAIK,gBAAe,CAAC,QAAQD,aAAYL,WAAU,CAAC,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,GAAG,GAAG;AACzF,QAAI,sBAAsB,CAAC;AAC3B,IAAAI,UAAS,qBAAqB;AAAA,MAC5B,sBAAsB,MAAM;AAAA,IAC9B,CAAC;AACD,WAAO,UAAUE,cAAa,mBAAmB;AACjD,QAAM,uBAAN,cAAmC,MAAM;AAAA,MACvC,YAAY,SAAS,OAAO;AAC1B,cAAM,OAAO;AACb,aAAK,OAAO;AACZ,aAAK,QAAQ;AAAA,MACf;AAAA,MACA,WAAW;AACT,YAAI,KAAK,OAAO;AACd,iBAAO,GAAG,KAAK,IAAI,KAAK,KAAK,OAAO,KAAK,KAAK,KAAK;AAAA,QACrD;AACA,eAAO,GAAG,KAAK,IAAI,KAAK,KAAK,OAAO;AAAA,MACtC;AAAA,IACF;AAAA;AAAA;","names":["__defProp","__getOwnPropDesc","__getOwnPropNames","__hasOwnProp","__export","__copyProps","__toCommonJS"]}
+{"version":3,"sources":["../../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js"],"sourcesContent":["\"use strict\";\nvar __defProp = Object.defineProperty;\nvar __getOwnPropDesc = Object.getOwnPropertyDescriptor;\nvar __getOwnPropNames = Object.getOwnPropertyNames;\nvar __hasOwnProp = Object.prototype.hasOwnProperty;\nvar __export = (target, all) => {\n  for (var name in all)\n    __defProp(target, name, { get: all[name], enumerable: true });\n};\nvar __copyProps = (to, from, except, desc) => {\n  if (from && typeof from === \"object\" || typeof from === \"function\") {\n    for (let key of __getOwnPropNames(from))\n      if (!__hasOwnProp.call(to, key) && key !== except)\n        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });\n  }\n  return to;\n};\nvar __toCommonJS = (mod) => __copyProps(__defProp({}, \"__esModule\", { value: true }), mod);\nvar token_error_exports = {};\n__export(token_error_exports, {\n  VercelOidcTokenError: () => VercelOidcTokenError\n});\nmodule.exports = __toCommonJS(token_error_exports);\nclass VercelOidcTokenError extends Error {\n  constructor(message, cause) {\n    super(message);\n    this.name = \"VercelOidcTokenError\";\n    this.cause = cause;\n  }\n  toString() {\n    if (this.cause) {\n      return `${this.name}: ${this.message}: ${this.cause}`;\n    }\n    return `${this.name}: ${this.message}`;\n  }\n}\n// Annotate the CommonJS export names for ESM import in node:\n0 && (module.exports = {\n  VercelOidcTokenError\n});\n"],"mappings":";;;;;;AAAA;AAAA;AAAA;AACA,QAAI,YAAY,OAAO;AACvB,QAAI,mBAAmB,OAAO;AAC9B,QAAI,oBAAoB,OAAO;AAC/B,QAAI,eAAe,OAAO,UAAU;AACpC,QAAI,WAAW,CAAC,QAAQ,QAAQ;AAC9B,eAAS,QAAQ;AACf,kBAAU,QAAQ,MAAM,EAAE,KAAK,IAAI,IAAI,GAAG,YAAY,KAAK,CAAC;AAAA,IAChE;AACA,QAAI,cAAc,CAAC,IAAI,MAAM,QAAQ,SAAS;AAC5C,UAAI,QAAQ,OAAO,SAAS,YAAY,OAAO,SAAS,YAAY;AAClE,iBAAS,OAAO,kBAAkB,IAAI;AACpC,cAAI,CAAC,aAAa,KAAK,IAAI,GAAG,KAAK,QAAQ;AACzC,sBAAU,IAAI,KAAK,EAAE,KAAK,MAAM,KAAK,GAAG,GAAG,YAAY,EAAE,OAAO,iBAAiB,MAAM,GAAG,MAAM,KAAK,WAAW,CAAC;AAAA,MACvH;AACA,aAAO;AAAA,IACT;AACA,QAAI,eAAe,CAAC,QAAQ,YAAY,UAAU,CAAC,GAAG,cAAc,EAAE,OAAO,KAAK,CAAC,GAAG,GAAG;AACzF,QAAI,sBAAsB,CAAC;AAC3B,aAAS,qBAAqB;AAAA,MAC5B,sBAAsB,MAAM;AAAA,IAC9B,CAAC;AACD,WAAO,UAAU,aAAa,mBAAmB;AACjD,QAAM,uBAAN,cAAmC,MAAM;AAAA,MACvC,YAAY,SAAS,OAAO;AAC1B,cAAM,OAAO;AACb,aAAK,OAAO;AACZ,aAAK,QAAQ;AAAA,MACf;AAAA,MACA,WAAW;AACT,YAAI,KAAK,OAAO;AACd,iBAAO,GAAG,KAAK,IAAI,KAAK,KAAK,OAAO,KAAK,KAAK,KAAK;AAAA,QACrD;AACA,eAAO,GAAG,KAAK,IAAI,KAAK,KAAK,OAAO;AAAA,MACtC;AAAA,IACF;AAAA;AAAA;","names":[]}

package/dist/{chunk-BKMQNEUD.js → chunk-5H446C7X.js}

@@ -39,55 +39,12 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
-// ../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js
-var require_token_error = __commonJS({
-  "../../node_modules/.bun/@vercel+oidc@3.0.5/node_modules/@vercel/oidc/dist/token-error.js"(exports, module) {
-    "use strict";
-    var __defProp2 = Object.defineProperty;
-    var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
-    var __getOwnPropNames2 = Object.getOwnPropertyNames;
-    var __hasOwnProp2 = Object.prototype.hasOwnProperty;
-    var __export2 = (target, all) => {
-      for (var name in all)
-        __defProp2(target, name, { get: all[name], enumerable: true });
-    };
-    var __copyProps2 = (to, from, except, desc) => {
-      if (from && typeof from === "object" || typeof from === "function") {
-        for (let key of __getOwnPropNames2(from))
-          if (!__hasOwnProp2.call(to, key) && key !== except)
-            __defProp2(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc2(from, key)) || desc.enumerable });
-      }
-      return to;
-    };
-    var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
-    var token_error_exports = {};
-    __export2(token_error_exports, {
-      VercelOidcTokenError: () => VercelOidcTokenError
-    });
-    module.exports = __toCommonJS2(token_error_exports);
-    var VercelOidcTokenError = class extends Error {
-      constructor(message, cause) {
-        super(message);
-        this.name = "VercelOidcTokenError";
-        this.cause = cause;
-      }
-      toString() {
-        if (this.cause) {
-          return `${this.name}: ${this.message}: ${this.cause}`;
-        }
-        return `${this.name}: ${this.message}`;
-      }
-    };
-  }
-});
-
 export {
   __require,
   __esm,
   __commonJS,
   __export,
   __toESM,
-  __toCommonJS,
-  require_token_error
+  __toCommonJS
 };
-//# sourceMappingURL=chunk-BKMQNEUD.js.map
+//# sourceMappingURL=chunk-5H446C7X.js.map