@agentv/core 2.18.4 → 3.0.0-next.1

package/dist/chunk-XBGLLO22.js

@@ -0,0 +1,65 @@
+// src/evaluation/providers/agentv-provider.ts
+import { createAnthropic } from "@ai-sdk/anthropic";
+import { createAzure } from "@ai-sdk/azure";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createOpenAI } from "@ai-sdk/openai";
+function parseModelString(model) {
+  const colonIndex = model.indexOf(":");
+  if (colonIndex === -1) {
+    throw new Error(
+      `Invalid model string "${model}". Expected format "provider:model" (e.g., "openai:gpt-5-mini")`
+    );
+  }
+  return {
+    provider: model.slice(0, colonIndex),
+    modelName: model.slice(colonIndex + 1)
+  };
+}
+function createLanguageModel(modelString) {
+  const { provider, modelName } = parseModelString(modelString);
+  switch (provider) {
+    case "openai":
+      return createOpenAI()(modelName);
+    case "anthropic":
+      return createAnthropic()(modelName);
+    case "azure":
+      return createAzure()(modelName);
+    case "google":
+      return createGoogleGenerativeAI()(modelName);
+    default:
+      throw new Error(
+        `Unsupported AI SDK provider "${provider}" in model string "${modelString}". Supported providers: openai, anthropic, azure, google`
+      );
+  }
+}
+var AgentvProvider = class {
+  id;
+  kind = "agentv";
+  targetName;
+  model;
+  constructor(targetName, config) {
+    this.id = `agentv:${targetName}`;
+    this.targetName = targetName;
+    this.model = createLanguageModel(config.model);
+  }
+  /**
+   * Direct invoke is not supported for the agentv provider.
+   * Use asLanguageModel() with generateText() instead.
+   */
+  async invoke(_request) {
+    throw new Error(
+      "AgentvProvider does not support direct invoke(). Use asLanguageModel() with generateText() instead."
+    );
+  }
+  /**
+   * Returns the resolved AI SDK LanguageModel for use with generateText/generateObject.
+   */
+  asLanguageModel() {
+    return this.model;
+  }
+};
+
+export {
+  AgentvProvider
+};
+//# sourceMappingURL=chunk-XBGLLO22.js.map

package/dist/chunk-XBGLLO22.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/evaluation/providers/agentv-provider.ts"],"sourcesContent":["import { createAnthropic } from '@ai-sdk/anthropic';\nimport { createAzure } from '@ai-sdk/azure';\nimport { createGoogleGenerativeAI } from '@ai-sdk/google';\nimport { createOpenAI } from '@ai-sdk/openai';\nimport type { LanguageModel } from 'ai';\n\nimport type { AgentVResolvedConfig } from './targets.js';\nimport type { Provider, ProviderRequest, ProviderResponse } from './types.js';\n\n/**\n * Parse a model string like \"openai:gpt-5-mini\" into provider prefix and model name.\n */\nfunction parseModelString(model: string): { provider: string; modelName: string } {\n  const colonIndex = model.indexOf(':');\n  if (colonIndex === -1) {\n    throw new Error(\n      `Invalid model string \"${model}\". Expected format \"provider:model\" (e.g., \"openai:gpt-5-mini\")`,\n    );\n  }\n  return {\n    provider: model.slice(0, colonIndex),\n    modelName: model.slice(colonIndex + 1),\n  };\n}\n\n/**\n * Create a LanguageModel from a model string using the appropriate AI SDK provider.\n */\nfunction createLanguageModel(modelString: string): LanguageModel {\n  const { provider, modelName } = parseModelString(modelString);\n\n  switch (provider) {\n    case 'openai':\n      // Cast: @ai-sdk/openai may return LanguageModelV3 while the rest of the\n      // codebase uses LanguageModelV2. The runtime API is compatible.\n      return createOpenAI()(modelName) as unknown as LanguageModel;\n    case 'anthropic':\n      return createAnthropic()(modelName);\n    case 'azure':\n      return createAzure()(modelName);\n    case 'google':\n      return createGoogleGenerativeAI()(modelName);\n    default:\n      throw new Error(\n        `Unsupported AI SDK provider \"${provider}\" in model string \"${modelString}\". Supported providers: openai, anthropic, azure, google`,\n      );\n  }\n}\n\n/**\n * AgentV built-in provider for LLM grader evaluation.\n *\n * Resolves an AI SDK model string (e.g., \"openai:gpt-5-mini\", \"anthropic:claude-sonnet-4-20250514\")\n * to a Vercel AI SDK LanguageModel by parsing the provider prefix and creating the appropriate\n * AI SDK provider directly. This provider is used exclusively for grader evaluation — it does not\n * support direct agent invocation.\n *\n * Usage: `--grader-target agentv --model openai:gpt-5-mini`\n */\nexport class AgentvProvider implements Provider {\n  readonly id: string;\n  readonly kind = 'agentv' as const;\n  readonly targetName: string;\n\n  private readonly model: LanguageModel;\n\n  constructor(targetName: string, config: AgentVResolvedConfig) {\n    this.id = `agentv:${targetName}`;\n    this.targetName = targetName;\n    this.model = createLanguageModel(config.model);\n  }\n\n  /**\n   * Direct invoke is not supported for the agentv provider.\n   * Use asLanguageModel() with generateText() instead.\n   */\n  async invoke(_request: ProviderRequest): Promise<ProviderResponse> {\n    throw new Error(\n      'AgentvProvider does not support direct invoke(). Use asLanguageModel() with generateText() instead.',\n    );\n  }\n\n  /**\n   * Returns the resolved AI SDK LanguageModel for use with generateText/generateObject.\n   */\n  asLanguageModel(): LanguageModel {\n    return this.model;\n  }\n}\n"],"mappings":";AAAA,SAAS,uBAAuB;AAChC,SAAS,mBAAmB;AAC5B,SAAS,gCAAgC;AACzC,SAAS,oBAAoB;AAS7B,SAAS,iBAAiB,OAAwD;AAChF,QAAM,aAAa,MAAM,QAAQ,GAAG;AACpC,MAAI,eAAe,IAAI;AACrB,UAAM,IAAI;AAAA,MACR,yBAAyB,KAAK;AAAA,IAChC;AAAA,EACF;AACA,SAAO;AAAA,IACL,UAAU,MAAM,MAAM,GAAG,UAAU;AAAA,IACnC,WAAW,MAAM,MAAM,aAAa,CAAC;AAAA,EACvC;AACF;AAKA,SAAS,oBAAoB,aAAoC;AAC/D,QAAM,EAAE,UAAU,UAAU,IAAI,iBAAiB,WAAW;AAE5D,UAAQ,UAAU;AAAA,IAChB,KAAK;AAGH,aAAO,aAAa,EAAE,SAAS;AAAA,IACjC,KAAK;AACH,aAAO,gBAAgB,EAAE,SAAS;AAAA,IACpC,KAAK;AACH,aAAO,YAAY,EAAE,SAAS;AAAA,IAChC,KAAK;AACH,aAAO,yBAAyB,EAAE,SAAS;AAAA,IAC7C;AACE,YAAM,IAAI;AAAA,QACR,gCAAgC,QAAQ,sBAAsB,WAAW;AAAA,MAC3E;AAAA,EACJ;AACF;AAYO,IAAM,iBAAN,MAAyC;AAAA,EACrC;AAAA,EACA,OAAO;AAAA,EACP;AAAA,EAEQ;AAAA,EAEjB,YAAY,YAAoB,QAA8B;AAC5D,SAAK,KAAK,UAAU,UAAU;AAC9B,SAAK,aAAa;AAClB,SAAK,QAAQ,oBAAoB,OAAO,KAAK;AAAA,EAC/C;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAO,UAAsD;AACjE,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,kBAAiC;AAC/B,WAAO,KAAK;AAAA,EACd;AACF;","names":[]}

package/dist/evaluation/validation/index.cjs

@@ -111,6 +111,8 @@ var import_yaml2 = require("yaml");
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
 var EVALUATOR_KIND_VALUES = [
+  "code-grader",
+  "llm-grader",
   "code-judge",
   "llm-judge",
   "rubric",
@@ -121,7 +123,7 @@ var EVALUATOR_KIND_VALUES = [
   "cost",
   "token-usage",
   "execution-metrics",
-  "agent-judge",
+  "skill-trigger",
   "contains",
   "contains-any",
   "contains-all",
@@ -133,7 +135,8 @@ var EVALUATOR_KIND_VALUES = [
   "regex",
   "is-json",
   "equals",
-  "rubrics"
+  "rubrics",
+  "inline-assert"
 ];
 var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
 function isEvaluatorKind(value) {
@@ -332,7 +335,7 @@ async function validateEvalFile(filePath) {
         });
       }
     }
-    const assertField = evalCase.assert;
+    const assertField = evalCase.assertions ?? evalCase.assert;
     if (assertField !== void 0) {
       validateAssertArray(assertField, location, absolutePath, errors);
     }
@@ -503,14 +506,14 @@ function validateAssertArray(assertField, parentLocation, filePath, errors) {
     errors.push({
       severity: "warning",
       filePath,
-      location: `${parentLocation}.assert`,
-      message: "'assert' must be an array of assertion objects."
+      location: `${parentLocation}.assertions`,
+      message: "'assertions' must be an array of assertion objects."
     });
     return;
   }
   for (let i = 0; i < assertField.length; i++) {
     const item = assertField[i];
-    const location = `${parentLocation}.assert[${i}]`;
+    const location = `${parentLocation}.assertions[${i}]`;
     if (!isObject(item)) {
       errors.push({
         severity: "warning",
@@ -671,7 +674,9 @@ var CliTargetInputSchema = import_zod.z.object({
   keep_output_files: import_zod.z.boolean().optional(),
   keepOutputFiles: import_zod.z.boolean().optional(),
   // Common target fields
+  grader_target: import_zod.z.string().optional(),
   judge_target: import_zod.z.string().optional(),
+  // backward compat
   workers: import_zod.z.number().int().min(1).optional(),
   provider_batching: import_zod.z.boolean().optional(),
   providerBatching: import_zod.z.boolean().optional()
@@ -711,7 +716,9 @@ var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set([
 var BASE_TARGET_SCHEMA = import_zod.z.object({
   name: import_zod.z.string().min(1, "target name is required"),
   provider: import_zod.z.string().min(1, "provider is required"),
+  grader_target: import_zod.z.string().optional(),
   judge_target: import_zod.z.string().optional(),
+  // backward compat
   workers: import_zod.z.number().int().min(1).optional(),
   workspace_template: import_zod.z.string().optional(),
   workspaceTemplate: import_zod.z.string().optional()
@@ -728,10 +735,13 @@ var KNOWN_PROVIDERS = [
   "pi-coding-agent",
   "pi-agent-sdk",
   "claude",
+  "claude-cli",
+  "claude-sdk",
   "cli",
   "mock",
   "vscode",
-  "vscode-insiders"
+  "vscode-insiders",
+  "agentv"
 ];
 var PROVIDER_ALIASES = [
   "azure-openai",
@@ -750,8 +760,6 @@ var PROVIDER_ALIASES = [
   // alias for "pi-coding-agent"
   "claude-code",
   // alias for "claude" (legacy)
-  "claude-sdk",
-  // alias for "claude"
   "openai",
   // legacy/future support
   "bedrock",
@@ -953,6 +961,7 @@ function getKnownSettings(provider) {
       return COPILOT_CLI_SETTINGS;
     case "claude":
     case "claude-code":
+    case "claude-cli":
     case "claude-sdk":
       return CLAUDE_SETTINGS;
     case "vscode":
@@ -972,7 +981,15 @@ function validateUnknownSettings(target, provider, absolutePath, location, error
   if (!knownSettings) {
     return;
   }
-  const baseFields = /* @__PURE__ */ new Set(["name", "provider", "judge_target", "workers", "$schema", "targets"]);
+  const baseFields = /* @__PURE__ */ new Set([
+    "name",
+    "provider",
+    "grader_target",
+    "judge_target",
+    "workers",
+    "$schema",
+    "targets"
+  ]);
   for (const key of Object.keys(target)) {
     if (removedTargetFields.has(key)) {
       errors.push({
@@ -1179,13 +1196,13 @@ async function validateTargetsFile(filePath) {
     if (typeof provider === "string") {
       validateUnknownSettings(target, provider, absolutePath, location, errors);
     }
-    const judgeTarget = target.judge_target;
-    if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {
+    const graderTarget = target.grader_target ?? target.judge_target;
+    if (graderTarget !== void 0 && typeof graderTarget !== "string") {
       errors.push({
         severity: "error",
         filePath: absolutePath,
-        location: `${location}.judge_target`,
-        message: "Invalid 'judge_target' field (must be a string)"
+        location: `${location}.grader_target`,
+        message: "Invalid 'grader_target' field (must be a string)"
       });
     }
   }