agentv 2.10.0 → 2.11.1

package/README.md

@@ -375,6 +375,69 @@ For complete examples and patterns, see:
 - [custom-evaluators](https://agentv.dev/evaluators/custom-evaluators/)
 - [code-judge-sdk example](examples/features/code-judge-sdk)
 
+### Deterministic Assertions
+
+Built-in assertion types for common text-matching patterns — no LLM judge or code_judge needed:
+
+| Type | Value | Behavior |
+|------|-------|----------|
+| `contains` | `string` | Pass if output includes the substring |
+| `contains_any` | `string[]` | Pass if output includes ANY of the strings |
+| `contains_all` | `string[]` | Pass if output includes ALL of the strings |
+| `icontains` | `string` | Case-insensitive `contains` |
+| `icontains_any` | `string[]` | Case-insensitive `contains_any` |
+| `icontains_all` | `string[]` | Case-insensitive `contains_all` |
+| `starts_with` | `string` | Pass if output starts with value (trimmed) |
+| `ends_with` | `string` | Pass if output ends with value (trimmed) |
+| `regex` | `string` | Pass if output matches regex (optional `flags: "i"`) |
+| `equals` | `string` | Pass if output exactly equals value (trimmed) |
+| `is_json` | — | Pass if output is valid JSON |
+
+All assertions support `weight`, `required`, and `negate` flags. Use `negate: true` to invert (no `not_` prefix needed).
+
+```yaml
+assert:
+  # Case-insensitive matching for natural language variation
+  - type: icontains_any
+    value: ["missing rule code", "need rule code", "provide rule code"]
+    required: true
+
+  # Multiple required terms
+  - type: icontains_all
+    value: ["country code", "rule codes"]
+
+  # Case-insensitive regex
+  - type: regex
+    value: "[a-z]+@[a-z]+\\.[a-z]+"
+    flags: "i"
+```
+
+See the [assert-extended example](examples/features/assert-extended) for complete patterns.
+
+### Target Configuration: `judge_target`
+
+Agent provider targets (`codex`, `copilot`, `claude`, `vscode`) **must** specify `judge_target` when using `llm_judge` or `rubrics` evaluators. Without it, AgentV errors at startup — agent providers can't return structured JSON for judging.
+
+```yaml
+targets:
+  # Agent target — requires judge_target for LLM-based evaluation
+  - name: codex_local
+    provider: codex
+    judge_target: azure_base  # Required: LLM provider for judging
+
+  # LLM target — no judge_target needed (judges itself)
+  - name: azure_base
+    provider: azure
+```
+
+### Agentic Eval Patterns
+
+When agents respond via tool calls instead of text, use `tool_trajectory` instead of text assertions:
+
+- **Agent takes workspace actions** (creates files, runs commands) → `tool_trajectory` evaluator
+- **Agent responds in text** (answers questions, asks for info) → `contains`/`icontains_any`/`llm_judge`
+- **Agent does both** → `composite` evaluator combining both
+
 ### LLM Judges
 
 Create markdown judge files with evaluation criteria and scoring guidelines:

package/dist/{chunk-G3OTPFYX.js → chunk-D6KWUG7C.js}

@@ -5,13 +5,14 @@ import {
   resolveEvalPaths,
   runEvalCommand,
   selectTarget,
-  toSnakeCaseDeep,
+  toSnakeCaseDeep as toSnakeCaseDeep2,
   validateConfigFile,
   validateEvalFile,
   validateFileReferences,
   validateTargetsFile
-} from "./chunk-PC3FAOHT.js";
+} from "./chunk-IL7CRMY6.js";
 import {
+  RepoManager,
   assembleLlmJudgePrompt,
   buildPromptInputs,
   createBuiltinRegistry,
@@ -22,9 +23,9 @@ import {
   loadTests,
   normalizeLineEndings,
   toCamelCaseDeep,
-  toSnakeCaseDeep as toSnakeCaseDeep2,
+  toSnakeCaseDeep,
   trimBaselineResult
-} from "./chunk-RJWTL3VS.js";
+} from "./chunk-MQIQH5LB.js";
 import {
   __commonJS,
   __esm,
@@ -2877,7 +2878,7 @@ function oneOf(literals) {
 // package.json
 var package_default = {
   name: "agentv",
-  version: "2.10.0",
+  version: "2.11.1",
   description: "CLI entry point for AgentV",
   type: "module",
   repository: {
@@ -2924,6 +2925,90 @@ var package_default = {
   }
 };
 
+// src/commands/cache/add.ts
+import { existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+var addCommand = command({
+  name: "add",
+  description: "Seed cache from a local git repository",
+  args: {
+    url: option({
+      long: "url",
+      description: "Remote URL to associate with the cache entry",
+      type: string
+    }),
+    from: option({
+      long: "from",
+      description: "Path to local git repository to clone from",
+      type: string
+    }),
+    force: flag({
+      long: "force",
+      short: "f",
+      description: "Overwrite existing cache entry"
+    })
+  },
+  handler: async ({ url, from, force }) => {
+    const localPath = resolve(from);
+    if (!existsSync(localPath)) {
+      console.error(`Error: local path does not exist: ${localPath}`);
+      process.exit(1);
+    }
+    if (!existsSync(join(localPath, ".git")) && !existsSync(join(localPath, "HEAD"))) {
+      console.error(`Error: ${localPath} does not appear to be a git repository`);
+      process.exit(1);
+    }
+    const manager = new RepoManager();
+    try {
+      const cachePath = await manager.seedCache(localPath, url, { force });
+      console.log(`Cache seeded from ${localPath}`);
+      console.log(`  Remote URL: ${url}`);
+      console.log(`  Cache path: ${cachePath}`);
+    } catch (err2) {
+      console.error(`Error: ${err2 instanceof Error ? err2.message : err2}`);
+      process.exit(1);
+    }
+  }
+});
+
+// src/commands/cache/index.ts
+var cleanCommand = command({
+  name: "clean",
+  description: "Remove all cached git repositories",
+  args: {
+    force: flag({
+      long: "force",
+      short: "f",
+      description: "Skip confirmation prompt"
+    })
+  },
+  handler: async ({ force }) => {
+    if (!force) {
+      const readline2 = await import("node:readline");
+      const rl = readline2.createInterface({ input: process.stdin, output: process.stdout });
+      const answer = await new Promise((resolve2) => {
+        rl.question("Remove all cached git repos from ~/.agentv/git-cache/? [y/N] ", resolve2);
+      });
+      rl.close();
+      if (answer.toLowerCase() !== "y") {
+        console.log("Cancelled.");
+        return;
+      }
+    }
+    const manager = new RepoManager();
+    await manager.cleanCache();
+    console.log("Cache cleaned.");
+  }
+});
+var cacheCommand = subcommands({
+  name: "cache",
+  description: "Manage AgentV cache",
+  cmds: {
+    add: addCommand,
+    clean: cleanCommand
+  }
+});
+
 // src/commands/compare/index.ts
 import { readFileSync } from "node:fs";
 var colors = {
@@ -3267,7 +3352,7 @@ var compareCommand = command({
         const results2 = loadJsonlResults(results[1]);
         const comparison = compareResults(results1, results2, effectiveThreshold);
         if (outputFormat === "json") {
-          console.log(JSON.stringify(toSnakeCaseDeep(comparison), null, 2));
+          console.log(JSON.stringify(toSnakeCaseDeep2(comparison), null, 2));
         } else {
           console.log(formatTable(comparison, results[0], results[1]));
         }
@@ -3313,7 +3398,7 @@ var compareCommand = command({
           }
           const comparison = compareResults(baselineResults, candidateResults, effectiveThreshold);
           if (outputFormat === "json") {
-            console.log(JSON.stringify(toSnakeCaseDeep(comparison), null, 2));
+            console.log(JSON.stringify(toSnakeCaseDeep2(comparison), null, 2));
           } else {
             console.log(formatTable(comparison, baseline, candidate));
           }
@@ -3322,7 +3407,7 @@ var compareCommand = command({
         } else {
           const matrixOutput = compareMatrix(groups, effectiveThreshold);
           if (outputFormat === "json") {
-            console.log(JSON.stringify(toSnakeCaseDeep(matrixOutput), null, 2));
+            console.log(JSON.stringify(toSnakeCaseDeep2(matrixOutput), null, 2));
           } else {
             console.log(formatMatrix(matrixOutput, baseline));
           }
@@ -3765,7 +3850,7 @@ async function processEvaluator(config, evalCase, candidate, promptInputs) {
         config: codeConfig.config ?? null
       };
       try {
-        const inputPayload = JSON.stringify(toSnakeCaseDeep2(payload), null, 2);
+        const inputPayload = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
         const stdout = await executeScript(script, inputPayload, 6e4, scriptCwd);
         const parsed = JSON.parse(stdout);
         return {
@@ -4042,7 +4127,7 @@ var evalRunCommand = command({
   },
   handler: async (args) => {
     if (args.evalPaths.length === 0 && process.stdin.isTTY) {
-      const { launchInteractiveWizard } = await import("./interactive-7KFUCBIP.js");
+      const { launchInteractiveWizard } = await import("./interactive-7NQRG7GK.js");
       await launchInteractiveWizard();
       return;
     }
@@ -4260,7 +4345,7 @@ var generateCommand = subcommands({
 });
 
 // src/commands/init/index.ts
-import { existsSync, mkdirSync, writeFileSync as writeFileSync2 } from "node:fs";
+import { existsSync as existsSync2, mkdirSync, writeFileSync as writeFileSync2 } from "node:fs";
 import path5 from "node:path";
 import * as readline from "node:readline/promises";
 
@@ -4330,14 +4415,14 @@ async function initCommand(options = {}) {
   const existingFiles = [];
   if (envTemplate) {
     const envFilePath = path5.join(targetPath, ".env.example");
-    if (existsSync(envFilePath)) {
+    if (existsSync2(envFilePath)) {
       existingFiles.push(".env.example");
     }
   }
-  if (existsSync(agentvDir)) {
+  if (existsSync2(agentvDir)) {
     for (const template of otherAgentvTemplates) {
       const targetFilePath = path5.join(agentvDir, template.path);
-      if (existsSync(targetFilePath)) {
+      if (existsSync2(targetFilePath)) {
         existingFiles.push(path5.relative(targetPath, targetFilePath));
       }
     }
@@ -4356,7 +4441,7 @@ async function initCommand(options = {}) {
     }
     console.log();
   }
-  if (!existsSync(agentvDir)) {
+  if (!existsSync2(agentvDir)) {
     mkdirSync(agentvDir, { recursive: true });
   }
   if (envTemplate) {
@@ -4367,7 +4452,7 @@ async function initCommand(options = {}) {
   for (const template of otherAgentvTemplates) {
     const targetFilePath = path5.join(agentvDir, template.path);
     const targetDirPath = path5.dirname(targetFilePath);
-    if (!existsSync(targetDirPath)) {
+    if (!existsSync2(targetDirPath)) {
       mkdirSync(targetDirPath, { recursive: true });
     }
     writeFileSync2(targetFilePath, template.content, "utf-8");
@@ -4421,7 +4506,7 @@ function detectPackageManager() {
   return detectPackageManagerFromPath(process.argv[1] ?? "");
 }
 function runCommand(cmd, args) {
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const child = spawn(cmd, args, { stdio: ["inherit", "pipe", "inherit"], shell: true });
     let stdout = "";
     child.stdout?.on("data", (data) => {
@@ -4429,7 +4514,7 @@ function runCommand(cmd, args) {
       stdout += data.toString();
     });
     child.on("error", reject);
-    child.on("close", (code) => resolve({ exitCode: code ?? 1, stdout }));
+    child.on("close", (code) => resolve2({ exitCode: code ?? 1, stdout }));
   });
 }
 var updateCommand = command({
@@ -4658,7 +4743,7 @@ var traceListCommand = command({
     try {
       const metas = listResultFiles(cwd, limit);
       if (outputFormat === "json") {
-        console.log(JSON.stringify(toSnakeCaseDeep(metas), null, 2));
+        console.log(JSON.stringify(toSnakeCaseDeep2(metas), null, 2));
       } else {
         console.log(formatListTable(metas));
       }
@@ -5325,7 +5410,7 @@ var traceStatsCommand = command({
       const groups = groupResults(results, groupBy2);
       if (outputFormat === "json") {
         const statsJson = computeStatsJson(groups, file);
-        console.log(JSON.stringify(toSnakeCaseDeep(statsJson), null, 2));
+        console.log(JSON.stringify(toSnakeCaseDeep2(statsJson), null, 2));
       } else {
         console.log(formatStatsTable(groups, file));
       }
@@ -5374,7 +5459,7 @@ var trimCommand = command({
         const record = JSON.parse(line);
         const camel = toCamelCaseDeep(record);
         const trimmed = trimBaselineResult(camel);
-        const snake = toSnakeCaseDeep2(trimmed);
+        const snake = toSnakeCaseDeep(trimmed);
         return JSON.stringify(snake);
       });
       const output = `${trimmedLines.join("\n")}
@@ -5596,12 +5681,100 @@ var validateCommand = command({
   }
 });
 
+// src/update-check.ts
+import { spawn as spawn2 } from "node:child_process";
+import { readFile as readFile3 } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join as join2 } from "node:path";
+var CHECK_INTERVAL_MS = 24 * 60 * 60 * 1e3;
+var AGENTV_DIR = join2(homedir(), ".agentv");
+var CACHE_FILE = "version-check.json";
+var NPM_REGISTRY_URL = "https://registry.npmjs.org/agentv/latest";
+async function getCachedUpdateInfo(path8) {
+  const filePath = path8 ?? join2(AGENTV_DIR, CACHE_FILE);
+  try {
+    const raw = await readFile3(filePath, "utf-8");
+    const data = JSON.parse(raw);
+    if (typeof data.latestVersion === "string" && typeof data.lastCheckedAt === "string") {
+      return data;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+function shouldCheck(cache) {
+  if (!cache) return true;
+  const elapsed = Date.now() - new Date(cache.lastCheckedAt).getTime();
+  return elapsed > CHECK_INTERVAL_MS;
+}
+function isNewer(a, b) {
+  const pa = a.split(".").map((s) => Number(s.replace(/-.*$/, "")));
+  const pb = b.split(".").map((s) => Number(s.replace(/-.*$/, "")));
+  for (let i = 0; i < 3; i++) {
+    if ((pa[i] ?? 0) > (pb[i] ?? 0)) return true;
+    if ((pa[i] ?? 0) < (pb[i] ?? 0)) return false;
+  }
+  return false;
+}
+function buildNotice(currentVersion, latestVersion) {
+  if (!latestVersion) return null;
+  if (!isNewer(latestVersion, currentVersion)) return null;
+  return `  Update available: ${currentVersion} \u2192 ${latestVersion}
+  Run \`agentv self update\` to upgrade.`;
+}
+function backgroundUpdateCheck() {
+  const dir = AGENTV_DIR;
+  const filePath = join2(dir, CACHE_FILE);
+  const script = `
+    const https = require('https');
+    const fs = require('fs');
+    const dir = ${JSON.stringify(dir)};
+    const filePath = ${JSON.stringify(filePath)};
+    https.get(${JSON.stringify(NPM_REGISTRY_URL)}, { timeout: 5000 }, (res) => {
+      if (res.statusCode !== 200) { res.resume(); process.exit(); }
+      let body = '';
+      res.on('data', (c) => body += c);
+      res.on('end', () => {
+        try {
+          const v = JSON.parse(body).version;
+          if (typeof v === 'string') {
+            fs.mkdirSync(dir, { recursive: true });
+            fs.writeFileSync(filePath, JSON.stringify({ latestVersion: v, lastCheckedAt: new Date().toISOString() }, null, 2));
+          }
+        } catch {}
+        process.exit();
+      });
+    }).on('error', () => process.exit()).on('timeout', function() { this.destroy(); process.exit(); });
+  `;
+  try {
+    const child = spawn2(process.execPath, ["-e", script], {
+      detached: true,
+      stdio: "ignore",
+      windowsHide: true
+    });
+    child.unref();
+  } catch {
+  }
+}
+async function getUpdateNotice(currentVersion) {
+  if (process.env.AGENTV_NO_UPDATE_CHECK === "1" || process.env.CI === "true") {
+    return null;
+  }
+  const cache = await getCachedUpdateInfo();
+  if (shouldCheck(cache)) {
+    backgroundUpdateCheck();
+  }
+  return buildNotice(currentVersion, cache?.latestVersion ?? null);
+}
+
 // src/index.ts
 var app = subcommands({
   name: "agentv",
   description: "AgentV CLI",
   version: package_default.version,
   cmds: {
+    cache: cacheCommand,
     eval: evalRunCommand,
     prompt: evalPromptCommand,
     compare: compareCommand,
@@ -5640,6 +5813,15 @@ function preprocessArgv(argv) {
   return result;
 }
 async function runCli(argv = process.argv) {
+  let updateNotice = null;
+  process.on("exit", () => {
+    if (updateNotice) process.stderr.write(`
+${updateNotice}
+`);
+  });
+  getUpdateNotice(package_default.version).then((n) => {
+    updateNotice = n;
+  });
   const processedArgv = preprocessArgv(argv);
   await run(binary(app), processedArgv);
 }
@@ -5649,4 +5831,4 @@ export {
   preprocessArgv,
   runCli
 };
-//# sourceMappingURL=chunk-G3OTPFYX.js.map
+//# sourceMappingURL=chunk-D6KWUG7C.js.map