@agentv/core 0.16.0 → 0.17.0

package/dist/index.d.cts

@@ -147,17 +147,6 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-/**
- * Formatting mode for segment content.
- * - 'agent': File references only (for providers with filesystem access)
- * - 'lm': Embedded file content with XML tags (for language model providers)
- */
-type FormattingMode = 'agent' | 'lm';
-/**
- * Extract fenced code blocks from AgentV user segments.
- */
-declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
-
 type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
 interface ChatMessage {
     readonly role: ChatMessageRole;
@@ -273,6 +262,17 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Formatting mode for segment content.
+ * - 'agent': File references only (for providers with filesystem access)
+ * - 'lm': Embedded file content with XML tags (for language model providers)
+ */
+type FormattingMode = "agent" | "lm";
+/**
+ * Extract fenced code blocks from AgentV user segments.
+ */
+declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
 /**
  * Build prompt inputs by consolidating user request context and guideline content.
  */

package/dist/index.d.ts

@@ -147,17 +147,6 @@ interface EvaluatorResult {
  */
 declare function getHitCount(result: Pick<EvaluationResult, "hits">): number;
 
-/**
- * Formatting mode for segment content.
- * - 'agent': File references only (for providers with filesystem access)
- * - 'lm': Embedded file content with XML tags (for language model providers)
- */
-type FormattingMode = 'agent' | 'lm';
-/**
- * Extract fenced code blocks from AgentV user segments.
- */
-declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
-
 type ChatMessageRole = "system" | "user" | "assistant" | "tool" | "function";
 interface ChatMessage {
     readonly role: ChatMessageRole;
@@ -273,6 +262,17 @@ interface TargetDefinition {
     readonly retryStatusCodes?: unknown | undefined;
 }
 
+/**
+ * Formatting mode for segment content.
+ * - 'agent': File references only (for providers with filesystem access)
+ * - 'lm': Embedded file content with XML tags (for language model providers)
+ */
+type FormattingMode = "agent" | "lm";
+/**
+ * Extract fenced code blocks from AgentV user segments.
+ */
+declare function extractCodeBlocks(segments: readonly JsonObject[]): readonly string[];
+
 /**
  * Build prompt inputs by consolidating user request context and guideline content.
  */

package/dist/index.js

@@ -1,5 +1,4 @@
 import {
-  TARGETS_SCHEMA_V2,
   buildDirectoryChain,
   buildSearchRoots,
   fileExists,
@@ -9,7 +8,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-IOCVST3R.js";
+} from "./chunk-YCIZ33BO.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -71,11 +70,11 @@ function extractCodeBlocks(segments) {
   const CODE_BLOCK_PATTERN = /```[\s\S]*?```/g;
   const codeBlocks = [];
   for (const segment of segments) {
-    const typeValue = segment["type"];
+    const typeValue = segment.type;
     if (typeof typeValue !== "string" || typeValue !== "text") {
       continue;
     }
-    const textValue = segment["value"];
+    const textValue = segment.value;
     if (typeof textValue !== "string") {
       continue;
     }
@@ -146,9 +145,9 @@ function asString(value) {
 }
 
 // src/evaluation/loaders/config-loader.ts
-import micromatch from "micromatch";
 import { readFile } from "node:fs/promises";
 import path2 from "node:path";
+import micromatch from "micromatch";
 import { parse } from "yaml";
 
 // src/evaluation/loaders/file-resolver.ts
@@ -290,8 +289,9 @@ Please add '$schema: ${SCHEMA_CONFIG_V2}' at the top of the file.`;
         guideline_patterns: guidelinePatterns
       };
     } catch (error) {
-      logWarning(`Could not read .agentv/config.yaml at ${configPath}: ${error.message}`);
-      continue;
+      logWarning(
+        `Could not read .agentv/config.yaml at ${configPath}: ${error.message}`
+      );
     }
   }
   return null;
@@ -334,9 +334,7 @@ var TEMPLATE_VARIABLES = {
   REFERENCE_ANSWER: "reference_answer",
   INPUT_MESSAGES: "input_messages"
 };
-var VALID_TEMPLATE_VARIABLES = new Set(
-  Object.values(TEMPLATE_VARIABLES)
-);
+var VALID_TEMPLATE_VARIABLES = new Set(Object.values(TEMPLATE_VARIABLES));
 var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
   TEMPLATE_VARIABLES.CANDIDATE_ANSWER,
   TEMPLATE_VARIABLES.EXPECTED_MESSAGES
@@ -353,13 +351,14 @@ function validateTemplateVariables(content, source) {
   const variablePattern = /\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g;
   const foundVariables = /* @__PURE__ */ new Set();
   const invalidVariables = [];
-  let match;
-  while ((match = variablePattern.exec(content)) !== null) {
+  let match = variablePattern.exec(content);
+  while (match !== null) {
     const varName = match[1];
     foundVariables.add(varName);
     if (!VALID_TEMPLATE_VARIABLES.has(varName)) {
       invalidVariables.push(varName);
     }
+    match = variablePattern.exec(content);
   }
   const hasCandidateAnswer = foundVariables.has(TEMPLATE_VARIABLES.CANDIDATE_ANSWER);
   const hasExpectedMessages = foundVariables.has(TEMPLATE_VARIABLES.EXPECTED_MESSAGES);
@@ -796,7 +795,14 @@ function needsRoleMarkers(messages, processedSegmentsByMessage) {
   return messagesWithContent > 1;
 }
 function buildChatPromptFromSegments(options) {
-  const { messages, segmentsByMessage, guidelinePatterns, guidelineContent, systemPrompt, mode = "lm" } = options;
+  const {
+    messages,
+    segmentsByMessage,
+    guidelinePatterns,
+    guidelineContent,
+    systemPrompt,
+    mode = "lm"
+  } = options;
   if (messages.length === 0) {
     return void 0;
   }
@@ -878,7 +884,6 @@ function logWarning4(message) {
 var ANSI_YELLOW6 = "\x1B[33m";
 var ANSI_RED = "\x1B[31m";
 var ANSI_RESET6 = "\x1B[0m";
-var SCHEMA_EVAL_V2 = "agentv-eval-v2";
 async function readTestSuiteMetadata(testFilePath) {
   try {
     const absolutePath = path6.resolve(testFilePath);
@@ -909,12 +914,6 @@ async function loadEvalCases(evalFilePath, repoRoot, options) {
   const datasetNameFromSuite = asString5(suite.dataset)?.trim();
   const fallbackDataset = path6.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
   const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
-  const schema = suite.$schema;
-  if (schema !== SCHEMA_EVAL_V2) {
-    const message = typeof schema === "string" ? `Invalid $schema value '${schema}' in ${evalFilePath}. Expected '${SCHEMA_EVAL_V2}'` : `Missing required field '$schema' in ${evalFilePath}.
-Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
-    throw new Error(message);
-  }
   const rawTestcases = suite.evalcases;
   if (!Array.isArray(rawTestcases)) {
     throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
@@ -938,11 +937,15 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
     const inputMessagesValue = evalcase.input_messages;
     const expectedMessagesValue = evalcase.expected_messages;
     if (!id || !outcome || !Array.isArray(inputMessagesValue)) {
-      logError(`Skipping incomplete eval case: ${id ?? "unknown"}. Missing required fields: id, outcome, and/or input_messages`);
+      logError(
+        `Skipping incomplete eval case: ${id ?? "unknown"}. Missing required fields: id, outcome, and/or input_messages`
+      );
       continue;
     }
     const hasExpectedMessages = Array.isArray(expectedMessagesValue) && expectedMessagesValue.length > 0;
-    const inputMessages = inputMessagesValue.filter((msg) => isTestMessage(msg));
+    const inputMessages = inputMessagesValue.filter(
+      (msg) => isTestMessage(msg)
+    );
     const expectedMessages = hasExpectedMessages ? expectedMessagesValue.filter((msg) => isTestMessage(msg)) : [];
     if (hasExpectedMessages && expectedMessages.length === 0) {
       logError(`No valid expected message found for eval case: ${id}`);
@@ -1615,7 +1618,7 @@ function formatTimeoutSuffix(timeoutMs) {
 import { exec as execCallback, spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { constants as constants2, createWriteStream } from "node:fs";
-import { access as access2, mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { access as access2, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path9 from "node:path";
 import { promisify as promisify2 } from "node:util";
@@ -1683,9 +1686,7 @@ function buildPromptDocument(request, inputFiles, options) {
     options?.guidelineOverrides
   );
   const inputFilesList = collectInputFiles(inputFiles);
-  const nonGuidelineInputFiles = inputFilesList.filter(
-    (file) => !guidelineFiles.includes(file)
-  );
+  const nonGuidelineInputFiles = inputFilesList.filter((file) => !guidelineFiles.includes(file));
   const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineInputFiles);
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
@@ -1857,7 +1858,15 @@ var CodexProvider = class {
     return path9.resolve(this.config.cwd);
   }
   buildCodexArgs() {
-    const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
+    const args = [
+      "--ask-for-approval",
+      "never",
+      "exec",
+      "--json",
+      "--color",
+      "never",
+      "--skip-git-repo-check"
+    ];
     if (this.config.args && this.config.args.length > 0) {
       args.push(...this.config.args);
     }
@@ -2481,7 +2490,12 @@ var MockProvider = class {
 
 // src/evaluation/providers/vscode.ts
 import path10 from "node:path";
-import { dispatchAgentSession, dispatchBatchAgent, getSubagentRoot, provisionSubagents } from "subagent";
+import {
+  dispatchAgentSession,
+  dispatchBatchAgent,
+  getSubagentRoot,
+  provisionSubagents
+} from "subagent";
 var VSCodeProvider = class {
   id;
   kind;
@@ -2598,9 +2612,7 @@ function buildPromptDocument2(request, attachments, guidelinePatterns) {
   }
   const guidelineFiles = collectGuidelineFiles2(attachments, guidelinePatterns);
   const attachmentFiles = collectAttachmentFiles(attachments);
-  const nonGuidelineAttachments = attachmentFiles.filter(
-    (file) => !guidelineFiles.includes(file)
-  );
+  const nonGuidelineAttachments = attachmentFiles.filter((file) => !guidelineFiles.includes(file));
   const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineAttachments);
   if (prereadBlock.length > 0) {
     parts.push("\n", prereadBlock);
@@ -2709,8 +2721,10 @@ async function ensureVSCodeSubagents(options) {
       if (result.skippedExisting.length > 0) {
         console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
       }
-      console.log(`
-total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
+      console.log(
+        `
+total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`
+      );
     }
     return {
       provisioned: true,
@@ -2736,27 +2750,6 @@ import { parse as parse3 } from "yaml";
 function isRecord(value) {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function checkSchema(parsed, absolutePath) {
-  const schema = parsed.$schema;
-  if (schema === void 0) {
-    throw new Error(
-      `Missing $schema field in targets.yaml at ${absolutePath}.
-Please add '$schema: ${TARGETS_SCHEMA_V2}' at the top of the file.`
-    );
-  }
-  if (typeof schema !== "string") {
-    throw new Error(
-      `Invalid $schema field in targets.yaml at ${absolutePath}.
-Expected a string value '${TARGETS_SCHEMA_V2}'.`
-    );
-  }
-  if (schema !== TARGETS_SCHEMA_V2) {
-    throw new Error(
-      `Invalid $schema '${schema}' in targets.yaml at ${absolutePath}.
-Expected '${TARGETS_SCHEMA_V2}'.`
-    );
-  }
-}
 function extractTargetsArray(parsed, absolutePath) {
   const targets = parsed.targets;
   if (!Array.isArray(targets)) {
@@ -2771,7 +2764,9 @@ function assertTargetDefinition(value, index, filePath) {
   const name = value.name;
   const provider = value.provider;
   if (typeof name !== "string" || name.trim().length === 0) {
-    throw new Error(`targets.yaml entry at index ${index} in ${filePath} is missing a valid 'name'`);
+    throw new Error(
+      `targets.yaml entry at index ${index} in ${filePath} is missing a valid 'name'`
+    );
   }
   if (typeof provider !== "string" || provider.trim().length === 0) {
     throw new Error(`targets.yaml entry '${name}' in ${filePath} is missing a valid 'provider'`);
@@ -2794,11 +2789,12 @@ async function readTargetDefinitions(filePath) {
   const raw = await readFile6(absolutePath, "utf8");
   const parsed = parse3(raw);
   if (!isRecord(parsed)) {
-    throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with '$schema' and 'targets' fields`);
+    throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
   }
-  checkSchema(parsed, absolutePath);
   const targets = extractTargetsArray(parsed, absolutePath);
-  const definitions = targets.map((entry, index) => assertTargetDefinition(entry, index, absolutePath));
+  const definitions = targets.map(
+    (entry, index) => assertTargetDefinition(entry, index, absolutePath)
+  );
   return definitions;
 }
 function listTargetNames(definitions) {
@@ -2875,7 +2871,11 @@ var LlmJudgeEvaluator = class {
     const formattedQuestion = context.promptInputs.question && context.promptInputs.question.trim().length > 0 ? context.promptInputs.question : context.evalCase.question;
     const variables = {
       [TEMPLATE_VARIABLES.INPUT_MESSAGES]: JSON.stringify(context.evalCase.input_segments, null, 2),
-      [TEMPLATE_VARIABLES.EXPECTED_MESSAGES]: JSON.stringify(context.evalCase.expected_segments, null, 2),
+      [TEMPLATE_VARIABLES.EXPECTED_MESSAGES]: JSON.stringify(
+        context.evalCase.expected_segments,
+        null,
+        2
+      ),
       [TEMPLATE_VARIABLES.CANDIDATE_ANSWER]: context.candidate.trim(),
       [TEMPLATE_VARIABLES.REFERENCE_ANSWER]: (context.evalCase.reference_answer ?? "").trim(),
       [TEMPLATE_VARIABLES.EXPECTED_OUTCOME]: context.evalCase.expected_outcome.trim(),
@@ -3121,7 +3121,7 @@ import { createHash, randomUUID as randomUUID2 } from "node:crypto";
 import { mkdir as mkdir2, writeFile as writeFile2 } from "node:fs/promises";
 import path12 from "node:path";
 
-// ../../node_modules/.pnpm/yocto-queue@1.2.1/node_modules/yocto-queue/index.js
+// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
 var Node = class {
   value;
   next;
@@ -3154,6 +3154,9 @@ var Queue = class {
     }
     this.#head = this.#head.next;
     this.#size--;
+    if (!this.#head) {
+      this.#tail = void 0;
+    }
     return current.value;
   }
   peek() {
@@ -3184,7 +3187,7 @@ var Queue = class {
   }
 };
 
-// ../../node_modules/.pnpm/p-limit@6.2.0/node_modules/p-limit/index.js
+// ../../node_modules/.bun/p-limit@6.2.0/node_modules/p-limit/index.js
 function pLimit(concurrency) {
   validateConcurrency(concurrency);
   const queue = new Queue();
@@ -3360,7 +3363,9 @@ async function runEvaluation(options) {
     } catch (error) {
       if (verbose) {
         const message = error instanceof Error ? error.message : String(error);
-        console.warn(`Provider batch execution failed, falling back to per-case dispatch: ${message}`);
+        console.warn(
+          `Provider batch execution failed, falling back to per-case dispatch: ${message}`
+        );
       }
     }
   }
@@ -3524,7 +3529,14 @@ async function runBatchEvaluation(options) {
         agentTimeoutMs
       });
     } catch (error) {
-      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs, provider);
+      const errorResult = buildErrorResult(
+        evalCase,
+        target.name,
+        nowFn(),
+        error,
+        promptInputs,
+        provider
+      );
       results.push(errorResult);
       if (onResult) {
         await onResult(errorResult);
@@ -3702,7 +3714,18 @@ async function evaluateCandidate(options) {
   };
 }
 async function runEvaluatorsForCase(options) {
-  const { evalCase, candidate, target, provider, evaluators, attempt, promptInputs, now, judgeProvider, agentTimeoutMs } = options;
+  const {
+    evalCase,
+    candidate,
+    target,
+    provider,
+    evaluators,
+    attempt,
+    promptInputs,
+    now,
+    judgeProvider,
+    agentTimeoutMs
+  } = options;
   if (evalCase.evaluators && evalCase.evaluators.length > 0) {
     return runEvaluatorList({
       evalCase,
@@ -3803,7 +3826,6 @@ async function runEvaluatorList(options) {
           reasoning: score2.reasoning,
           evaluator_provider_request: score2.evaluatorRawRequest
         });
-        continue;
       }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
@@ -3814,7 +3836,11 @@ async function runEvaluatorList(options) {
         expectedAspectCount: 1,
         reasoning: message
       };
-      scored.push({ score: fallbackScore, name: evaluator.name ?? "unknown", type: evaluator.type ?? "unknown" });
+      scored.push({
+        score: fallbackScore,
+        name: evaluator.name ?? "unknown",
+        type: evaluator.type ?? "unknown"
+      });
       evaluatorResults.push({
         name: evaluator.name ?? "unknown",
         type: evaluator.type ?? "unknown",
@@ -3828,7 +3854,10 @@ async function runEvaluatorList(options) {
   const aggregateScore = scored.length > 0 ? scored.reduce((total, entry) => total + entry.score.score, 0) / scored.length : 0;
   const hits = scored.flatMap((entry) => entry.score.hits);
   const misses = scored.flatMap((entry) => entry.score.misses);
-  const expectedAspectCount = scored.reduce((total, entry) => total + (entry.score.expectedAspectCount ?? 0), 0);
+  const expectedAspectCount = scored.reduce(
+    (total, entry) => total + (entry.score.expectedAspectCount ?? 0),
+    0
+  );
   const rawAspects = scored.flatMap((entry) => entry.score.rawAspects ?? []);
   const reasoningParts = scored.map((entry) => entry.score.reasoning ? `${entry.name}: ${entry.score.reasoning}` : void 0).filter(isNonEmptyString2);
   const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
@@ -3843,7 +3872,18 @@ async function runEvaluatorList(options) {
   return { score, evaluatorResults };
 }
 async function runLlmJudgeEvaluator(options) {
-  const { config, evalCase, candidate, target, provider, evaluatorRegistry, attempt, promptInputs, now, judgeProvider } = options;
+  const {
+    config,
+    evalCase,
+    candidate,
+    target,
+    provider,
+    evaluatorRegistry,
+    attempt,
+    promptInputs,
+    now,
+    judgeProvider
+  } = options;
   const customPrompt = await resolveCustomPrompt(config);
   return evaluatorRegistry.llm_judge.evaluate({
     evalCase,