@agentv/core 0.2.8 → 0.2.11

package/dist/index.d.cts

@@ -202,6 +202,15 @@ interface Provider {
     readonly kind: ProviderKind;
     readonly targetName: string;
     invoke(request: ProviderRequest): Promise<ProviderResponse>;
+    /**
+     * Optional capability marker for provider-managed batching (single session handling multiple requests).
+     */
+    readonly supportsBatch?: boolean;
+    /**
+     * Optional batch invocation hook. When defined alongside supportsBatch=true,
+     * the orchestrator may send multiple requests in a single provider session.
+     */
+    invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -251,30 +260,35 @@ type ResolvedTarget = {
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
     readonly kind: "anthropic";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
     readonly kind: "gemini";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
 } | {
     readonly kind: "mock";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
     readonly kind: "vscode" | "vscode-insiders";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
 };
 declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;

package/dist/index.d.ts

@@ -202,6 +202,15 @@ interface Provider {
     readonly kind: ProviderKind;
     readonly targetName: string;
     invoke(request: ProviderRequest): Promise<ProviderResponse>;
+    /**
+     * Optional capability marker for provider-managed batching (single session handling multiple requests).
+     */
+    readonly supportsBatch?: boolean;
+    /**
+     * Optional batch invocation hook. When defined alongside supportsBatch=true,
+     * the orchestrator may send multiple requests in a single provider session.
+     */
+    invokeBatch?(requests: readonly ProviderRequest[]): Promise<readonly ProviderResponse[]>;
 }
 type EnvLookup = Readonly<Record<string, string | undefined>>;
 interface TargetDefinition {
@@ -251,30 +260,35 @@ type ResolvedTarget = {
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AzureResolvedConfig;
 } | {
     readonly kind: "anthropic";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: AnthropicResolvedConfig;
 } | {
     readonly kind: "gemini";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: GeminiResolvedConfig;
 } | {
     readonly kind: "mock";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: MockResolvedConfig;
 } | {
     readonly kind: "vscode" | "vscode-insiders";
     readonly name: string;
     readonly judgeTarget?: string;
     readonly workers?: number;
+    readonly providerBatching?: boolean;
     readonly config: VSCodeResolvedConfig;
 };
 declare function resolveTargetDefinition(definition: TargetDefinition, env?: EnvLookup): ResolvedTarget;

package/dist/index.js

@@ -5,7 +5,7 @@ import {
   fileExists,
   findGitRoot,
   resolveFileReference
-} from "./chunk-XXNQA4EW.js";
+} from "./chunk-P4GOYWYH.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -735,6 +735,9 @@ function normalizeAzureApiVersion(value) {
 function resolveTargetDefinition(definition, env = process.env) {
   const parsed = BASE_TARGET_SCHEMA.parse(definition);
   const provider = parsed.provider.toLowerCase();
+  const providerBatching = resolveOptionalBoolean(
+    parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
+  );
   switch (provider) {
     case "azure":
     case "azure-openai":
@@ -743,6 +746,7 @@ function resolveTargetDefinition(definition, env = process.env) {
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
+        providerBatching,
         config: resolveAzureConfig(parsed, env)
       };
     case "anthropic":
@@ -751,6 +755,7 @@ function resolveTargetDefinition(definition, env = process.env) {
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
+        providerBatching,
         config: resolveAnthropicConfig(parsed, env)
       };
     case "gemini":
@@ -761,6 +766,7 @@ function resolveTargetDefinition(definition, env = process.env) {
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
+        providerBatching,
         config: resolveGeminiConfig(parsed, env)
       };
     case "mock":
@@ -769,6 +775,7 @@ function resolveTargetDefinition(definition, env = process.env) {
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
+        providerBatching,
         config: resolveMockConfig(parsed)
       };
     case "vscode":
@@ -778,6 +785,7 @@ function resolveTargetDefinition(definition, env = process.env) {
         name: parsed.name,
         judgeTarget: parsed.judge_target,
         workers: parsed.workers,
+        providerBatching,
         config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
       };
     default:
@@ -964,11 +972,17 @@ function isLikelyEnvReference(value) {
 // src/evaluation/providers/vscode.ts
 import { readFile as readFile2 } from "node:fs/promises";
 import path2 from "node:path";
-import { dispatchAgentSession, getSubagentRoot, provisionSubagents } from "subagent";
+import {
+  dispatchAgentSession,
+  dispatchBatchAgent,
+  getSubagentRoot,
+  provisionSubagents
+} from "subagent";
 var VSCodeProvider = class {
   id;
   kind;
   targetName;
+  supportsBatch = true;
   config;
   constructor(targetName, config, kind) {
     this.id = `${kind}:${targetName}`;
@@ -1015,38 +1029,102 @@ var VSCodeProvider = class {
       }
     };
   }
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    const normalizedRequests = requests.map((req) => ({
+      request: req,
+      attachments: normalizeAttachments(req.attachments)
+    }));
+    const combinedAttachments = mergeAttachments(
+      normalizedRequests.map(({ attachments }) => attachments)
+    );
+    const userQueries = normalizedRequests.map(
+      ({ request, attachments }) => buildPromptDocument(request, attachments, request.guideline_patterns)
+    );
+    const session = await dispatchBatchAgent({
+      userQueries,
+      extraAttachments: combinedAttachments,
+      wait: this.config.waitForResponse,
+      dryRun: this.config.dryRun,
+      vscodeCmd: this.config.command,
+      subagentRoot: this.config.subagentRoot,
+      workspaceTemplate: this.config.workspaceTemplate,
+      silent: true
+    });
+    if (session.exitCode !== 0 || !session.responseFiles) {
+      const failure = session.error ?? "VS Code subagent did not produce batch responses";
+      throw new Error(failure);
+    }
+    if (this.config.dryRun) {
+      return normalizedRequests.map(({ attachments }) => ({
+        text: "",
+        raw: {
+          session,
+          attachments,
+          allAttachments: combinedAttachments
+        }
+      }));
+    }
+    if (session.responseFiles.length !== requests.length) {
+      throw new Error(
+        `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
+      );
+    }
+    const responses = [];
+    for (const [index, responseFile] of session.responseFiles.entries()) {
+      const responseText = await readFile2(responseFile, "utf8");
+      responses.push({
+        text: responseText,
+        raw: {
+          session,
+          attachments: normalizedRequests[index]?.attachments,
+          allAttachments: combinedAttachments,
+          responseFile
+        }
+      });
+    }
+    return responses;
+  }
 };
 function buildPromptDocument(request, attachments, guidelinePatterns) {
   const parts = [];
   const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
-  if (guidelineFiles.length > 0) {
-    parts.push("\n", buildMandatoryPrereadBlock(guidelineFiles));
+  const attachmentFiles = collectAttachmentFiles(attachments);
+  const nonGuidelineAttachments = attachmentFiles.filter(
+    (file) => !guidelineFiles.includes(file)
+  );
+  const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
+  if (prereadBlock.length > 0) {
+    parts.push("\n", prereadBlock);
   }
   parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
   return parts.join("\n").trim();
 }
-function buildMandatoryPrereadBlock(guidelineFiles) {
-  if (guidelineFiles.length === 0) {
+function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
+  if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
     return "";
   }
-  const fileList = [];
-  let counter = 0;
-  for (const absolutePath of guidelineFiles) {
-    counter += 1;
+  const buildList = (files) => files.map((absolutePath) => {
     const fileName = path2.basename(absolutePath);
     const fileUri = pathToFileUri(absolutePath);
-    fileList.push(`* [${fileName}](${fileUri})`);
-  }
-  const filesText = fileList.join("\n");
-  const instruction = [
-    `Read all guideline files:
-${filesText}.
-`,
-    `If any file is missing, fail with ERROR: missing-file <filename> and stop.
-`,
-    `Then apply system_instructions on the user query below.`
-  ].join("");
-  return `${instruction}`;
+    return `* [${fileName}](${fileUri})`;
+  });
+  const sections = [];
+  if (guidelineFiles.length > 0) {
+    sections.push(`Read all guideline files:
+${buildList(guidelineFiles).join("\n")}.`);
+  }
+  if (attachmentFiles.length > 0) {
+    sections.push(`Read all attachment files:
+${buildList(attachmentFiles).join("\n")}.`);
+  }
+  sections.push(
+    "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
+    "Then apply system_instructions on the user query below."
+  );
+  return sections.join("\n");
 }
 function collectGuidelineFiles(attachments, guidelinePatterns) {
   if (!attachments || attachments.length === 0) {
@@ -1064,6 +1142,19 @@ function collectGuidelineFiles(attachments, guidelinePatterns) {
   }
   return Array.from(unique.values());
 }
+function collectAttachmentFiles(attachments) {
+  if (!attachments || attachments.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    const absolutePath = path2.resolve(attachment);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(unique.values());
+}
 function pathToFileUri(filePath) {
   const absolutePath = path2.isAbsolute(filePath) ? filePath : path2.resolve(filePath);
   const normalizedPath = absolutePath.replace(/\\/g, "/");
@@ -1082,6 +1173,16 @@ function normalizeAttachments(attachments) {
   }
   return Array.from(deduped);
 }
+function mergeAttachments(all) {
+  const deduped = /* @__PURE__ */ new Set();
+  for (const list of all) {
+    if (!list) continue;
+    for (const attachment of list) {
+      deduped.add(path2.resolve(attachment));
+    }
+  }
+  return deduped.size > 0 ? Array.from(deduped) : void 0;
+}
 async function ensureVSCodeSubagents(options) {
   const { kind, count, verbose = false } = options;
   const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
@@ -1811,6 +1912,12 @@ async function runEvaluation(options) {
   };
   const graderRegistry = buildGraderRegistry(graders, resolveJudgeProvider);
   const primaryProvider = getOrCreateProvider(target);
+  const providerSupportsBatch = target.providerBatching === true && primaryProvider.supportsBatch === true && typeof primaryProvider.invokeBatch === "function";
+  if (target.providerBatching && !providerSupportsBatch && verbose) {
+    console.warn(
+      `Provider batching requested for target '${target.name}', but provider does not advertise batch support. Using per-case dispatch.`
+    );
+  }
   if (onProgress && filteredEvalCases.length > 0) {
     for (let i = 0; i < filteredEvalCases.length; i++) {
       await onProgress({
@@ -1820,6 +1927,27 @@ async function runEvaluation(options) {
       });
     }
   }
+  if (providerSupportsBatch) {
+    try {
+      return await runBatchEvaluation({
+        evalCases: filteredEvalCases,
+        provider: primaryProvider,
+        target,
+        graderRegistry,
+        promptDumpDir,
+        nowFn: now ?? (() => /* @__PURE__ */ new Date()),
+        onProgress,
+        onResult,
+        verbose,
+        resolveJudgeProvider
+      });
+    } catch (error) {
+      if (verbose) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.warn(`Provider batch execution failed, falling back to per-case dispatch: ${message}`);
+      }
+    }
+  }
   const workers = options.maxConcurrency ?? target.workers ?? 1;
   const limit = pLimit(workers);
   let nextWorkerId = 1;
@@ -1903,6 +2031,137 @@ async function runEvaluation(options) {
   }
   return results;
 }
+async function runBatchEvaluation(options) {
+  const {
+    evalCases,
+    provider,
+    target,
+    graderRegistry,
+    promptDumpDir,
+    nowFn,
+    onProgress,
+    onResult,
+    resolveJudgeProvider
+  } = options;
+  const promptInputsList = [];
+  for (const evalCase of evalCases) {
+    const promptInputs = await buildPromptInputs(evalCase);
+    if (promptDumpDir) {
+      await dumpPrompt(promptDumpDir, evalCase, promptInputs);
+    }
+    promptInputsList.push(promptInputs);
+  }
+  const batchRequests = evalCases.map((evalCase, index) => {
+    const promptInputs = promptInputsList[index];
+    return {
+      prompt: promptInputs.request,
+      guidelines: promptInputs.guidelines,
+      guideline_patterns: evalCase.guideline_patterns,
+      attachments: evalCase.file_paths,
+      evalCaseId: evalCase.id,
+      metadata: {
+        systemPrompt: promptInputs.systemMessage ?? ""
+      }
+    };
+  });
+  const batchResponse = await provider.invokeBatch?.(batchRequests);
+  if (!Array.isArray(batchResponse)) {
+    throw new Error("Provider batching failed: invokeBatch did not return an array");
+  }
+  if (batchResponse.length !== evalCases.length) {
+    throw new Error(
+      `Provider batching failed: expected ${evalCases.length} responses, received ${batchResponse.length}`
+    );
+  }
+  if (onProgress) {
+    const startedAt = Date.now();
+    for (let i = 0; i < evalCases.length; i++) {
+      await onProgress({
+        workerId: 1,
+        evalId: evalCases[i].id,
+        status: "running",
+        startedAt
+      });
+    }
+  }
+  const results = [];
+  for (let i = 0; i < evalCases.length; i++) {
+    const evalCase = evalCases[i];
+    const promptInputs = promptInputsList[i];
+    const providerResponse = batchResponse[i];
+    const now = nowFn();
+    const graderKind = evalCase.grader ?? "heuristic";
+    const activeGrader = graderRegistry[graderKind] ?? graderRegistry.heuristic;
+    if (!activeGrader) {
+      throw new Error(`No grader registered for kind '${graderKind}'`);
+    }
+    let grade;
+    try {
+      grade = await activeGrader.grade({
+        evalCase,
+        candidate: providerResponse.text ?? "",
+        target,
+        provider,
+        attempt: 0,
+        promptInputs,
+        now,
+        judgeProvider: await resolveJudgeProvider(target)
+      });
+    } catch (error) {
+      const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
+      results.push(errorResult);
+      if (onResult) {
+        await onResult(errorResult);
+      }
+      if (onProgress) {
+        await onProgress({
+          workerId: 1,
+          evalId: evalCase.id,
+          status: "failed",
+          completedAt: Date.now(),
+          error: error instanceof Error ? error.message : String(error)
+        });
+      }
+      continue;
+    }
+    const completedAt = nowFn();
+    const rawRequest = {
+      request: promptInputs.request,
+      guidelines: promptInputs.guidelines,
+      guideline_paths: evalCase.guideline_paths,
+      system_message: promptInputs.systemMessage ?? ""
+    };
+    const result = {
+      eval_id: evalCase.id,
+      conversation_id: evalCase.conversation_id,
+      score: grade.score,
+      hits: grade.hits,
+      misses: grade.misses,
+      model_answer: providerResponse.text ?? "",
+      expected_aspect_count: grade.expectedAspectCount,
+      target: target.name,
+      timestamp: completedAt.toISOString(),
+      reasoning: grade.reasoning,
+      raw_aspects: grade.rawAspects,
+      raw_request: rawRequest,
+      grader_raw_request: grade.graderRawRequest
+    };
+    results.push(result);
+    if (onResult) {
+      await onResult(result);
+    }
+    if (onProgress) {
+      await onProgress({
+        workerId: 1,
+        evalId: evalCase.id,
+        status: "completed",
+        startedAt: 0,
+        completedAt: Date.now()
+      });
+    }
+  }
+  return results;
+}
 async function runEvalCase(options) {
   const {
     evalCase,