@agentv/core 0.2.3

package/dist/index.cjs

@@ -0,0 +1,2204 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  GRADER_KINDS: () => GRADER_KINDS,
+  HeuristicGrader: () => HeuristicGrader,
+  QualityGrader: () => QualityGrader,
+  TEST_MESSAGE_ROLES: () => TEST_MESSAGE_ROLES,
+  buildPromptInputs: () => buildPromptInputs,
+  calculateHits: () => calculateHits,
+  calculateMisses: () => calculateMisses,
+  createAgentKernel: () => createAgentKernel,
+  createProvider: () => createProvider,
+  ensureVSCodeSubagents: () => ensureVSCodeSubagents,
+  extractAspects: () => extractAspects,
+  extractCodeBlocks: () => extractCodeBlocks,
+  getHitCount: () => getHitCount,
+  isErrorLike: () => isErrorLike,
+  isGraderKind: () => isGraderKind,
+  isGuidelineFile: () => isGuidelineFile,
+  isJsonObject: () => isJsonObject,
+  isJsonValue: () => isJsonValue,
+  isTestMessage: () => isTestMessage,
+  isTestMessageRole: () => isTestMessageRole,
+  listTargetNames: () => listTargetNames,
+  loadTestCases: () => loadTestCases,
+  readTargetDefinitions: () => readTargetDefinitions,
+  resolveAndCreateProvider: () => resolveAndCreateProvider,
+  resolveTargetDefinition: () => resolveTargetDefinition,
+  runEvaluation: () => runEvaluation,
+  runTestCase: () => runTestCase,
+  scoreCandidateResponse: () => scoreCandidateResponse
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/evaluation/types.ts
+var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
+var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
+var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
+function isTestMessageRole(value) {
+  return typeof value === "string" && TEST_MESSAGE_ROLE_SET.has(value);
+}
+function isJsonObject(value) {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return false;
+  }
+  return Object.values(value).every(isJsonValue);
+}
+function isJsonValue(value) {
+  if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return true;
+  }
+  if (Array.isArray(value)) {
+    return value.every(isJsonValue);
+  }
+  if (typeof value === "object") {
+    return isJsonObject(value);
+  }
+  return false;
+}
+function isTestMessage(value) {
+  if (typeof value !== "object" || value === null) {
+    return false;
+  }
+  const candidate = value;
+  if (!isTestMessageRole(candidate.role)) {
+    return false;
+  }
+  if (typeof candidate.content === "string") {
+    return true;
+  }
+  if (!Array.isArray(candidate.content)) {
+    return false;
+  }
+  return candidate.content.every(isJsonObject);
+}
+var GRADER_KIND_VALUES = ["heuristic", "llm_judge"];
+var GRADER_KINDS = GRADER_KIND_VALUES;
+var GRADER_KIND_SET = new Set(GRADER_KIND_VALUES);
+function isGraderKind(value) {
+  return typeof value === "string" && GRADER_KIND_SET.has(value);
+}
+function getHitCount(result) {
+  return result.hits.length;
+}
+
+// src/evaluation/yaml-parser.ts
+var import_node_fs2 = require("fs");
+var import_promises2 = require("fs/promises");
+var import_node_path2 = __toESM(require("path"), 1);
+var import_node_url = require("url");
+var import_yaml = require("yaml");
+
+// src/evaluation/file-utils.ts
+var import_node_fs = require("fs");
+var import_promises = require("fs/promises");
+var import_node_path = __toESM(require("path"), 1);
+async function fileExists(filePath) {
+  try {
+    await (0, import_promises.access)(filePath, import_node_fs.constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function buildSearchRoots(evalPath, repoRoot) {
+  const uniqueRoots = [];
+  const addRoot = (root) => {
+    const normalized = import_node_path.default.resolve(root);
+    if (!uniqueRoots.includes(normalized)) {
+      uniqueRoots.push(normalized);
+    }
+  };
+  let currentDir = import_node_path.default.dirname(evalPath);
+  let reachedBoundary = false;
+  while (!reachedBoundary) {
+    addRoot(currentDir);
+    const parentDir = import_node_path.default.dirname(currentDir);
+    if (currentDir === repoRoot || parentDir === currentDir) {
+      reachedBoundary = true;
+    } else {
+      currentDir = parentDir;
+    }
+  }
+  addRoot(repoRoot);
+  addRoot(process.cwd());
+  return uniqueRoots;
+}
+function trimLeadingSeparators(value) {
+  const trimmed = value.replace(/^[/\\]+/, "");
+  return trimmed.length > 0 ? trimmed : value;
+}
+async function resolveFileReference(rawValue, searchRoots) {
+  const displayPath = trimLeadingSeparators(rawValue);
+  const potentialPaths = [];
+  if (import_node_path.default.isAbsolute(rawValue)) {
+    potentialPaths.push(import_node_path.default.normalize(rawValue));
+  }
+  for (const base of searchRoots) {
+    potentialPaths.push(import_node_path.default.resolve(base, displayPath));
+  }
+  const attempted = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const candidate of potentialPaths) {
+    const absoluteCandidate = import_node_path.default.resolve(candidate);
+    if (seen.has(absoluteCandidate)) {
+      continue;
+    }
+    seen.add(absoluteCandidate);
+    attempted.push(absoluteCandidate);
+    if (await fileExists(absoluteCandidate)) {
+      return { displayPath, resolvedPath: absoluteCandidate, attempted };
+    }
+  }
+  return { displayPath, attempted };
+}
+
+// src/evaluation/yaml-parser.ts
+var CODE_BLOCK_PATTERN = /```[\s\S]*?```/g;
+var ANSI_YELLOW = "\x1B[33m";
+var ANSI_RESET = "\x1B[0m";
+var SCHEMA_EVAL_V2 = "agentv-eval-v2";
+function isGuidelineFile(filePath) {
+  const normalized = filePath.split("\\").join("/");
+  return normalized.endsWith(".instructions.md") || normalized.includes("/instructions/") || normalized.endsWith(".prompt.md") || normalized.includes("/prompts/");
+}
+function extractCodeBlocks(segments) {
+  const codeBlocks = [];
+  for (const segment of segments) {
+    const typeValue = segment["type"];
+    if (typeof typeValue !== "string" || typeValue !== "text") {
+      continue;
+    }
+    const textValue = segment["value"];
+    if (typeof textValue !== "string") {
+      continue;
+    }
+    const matches = textValue.match(CODE_BLOCK_PATTERN);
+    if (matches) {
+      codeBlocks.push(...matches);
+    }
+  }
+  return codeBlocks;
+}
+async function loadTestCases(testFilePath, repoRoot, options) {
+  const verbose = options?.verbose ?? false;
+  const absoluteTestPath = import_node_path2.default.resolve(testFilePath);
+  if (!await fileExists2(absoluteTestPath)) {
+    throw new Error(`Test file not found: ${testFilePath}`);
+  }
+  const repoRootPath = resolveToAbsolutePath(repoRoot);
+  const searchRoots = buildSearchRoots(absoluteTestPath, repoRootPath);
+  const rawFile = await (0, import_promises2.readFile)(absoluteTestPath, "utf8");
+  const parsed = (0, import_yaml.parse)(rawFile);
+  if (!isJsonObject(parsed)) {
+    throw new Error(`Invalid test file format: ${testFilePath}`);
+  }
+  const suite = parsed;
+  const schema = suite.$schema;
+  if (schema !== SCHEMA_EVAL_V2) {
+    const message = typeof schema === "string" ? `Invalid $schema value '${schema}' in ${testFilePath}. Expected '${SCHEMA_EVAL_V2}'` : `Missing required field '$schema' in ${testFilePath}.
+Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
+    throw new Error(message);
+  }
+  const rawTestcases = suite.evalcases;
+  if (!Array.isArray(rawTestcases)) {
+    throw new Error(`Invalid test file format: ${testFilePath} - missing 'evalcases' field`);
+  }
+  const globalGrader = coerceGrader(suite.grader) ?? "llm_judge";
+  const results = [];
+  for (const rawTestcase of rawTestcases) {
+    if (!isJsonObject(rawTestcase)) {
+      logWarning("Skipping invalid test case entry (expected object)");
+      continue;
+    }
+    const testcase = rawTestcase;
+    const id = asString(testcase.id);
+    const conversationId = asString(testcase.conversation_id);
+    const outcome = asString(testcase.outcome);
+    const inputMessagesValue = testcase.input_messages;
+    const expectedMessagesValue = testcase.expected_messages;
+    if (!id || !outcome || !Array.isArray(inputMessagesValue)) {
+      logWarning(`Skipping incomplete test case: ${id ?? "unknown"}`);
+      continue;
+    }
+    if (!Array.isArray(expectedMessagesValue)) {
+      logWarning(`Test case '${id}' missing expected_messages array`);
+      continue;
+    }
+    const inputMessages = inputMessagesValue.filter((msg) => isTestMessage(msg));
+    const expectedMessages = expectedMessagesValue.filter((msg) => isTestMessage(msg));
+    const assistantMessages = expectedMessages.filter((message) => message.role === "assistant");
+    const userMessages = inputMessages.filter((message) => message.role === "user");
+    if (assistantMessages.length === 0) {
+      logWarning(`No assistant message found for test case: ${id}`);
+      continue;
+    }
+    if (assistantMessages.length > 1) {
+      logWarning(`Multiple assistant messages found for test case: ${id}, using first`);
+    }
+    const userSegments = [];
+    const guidelinePaths = [];
+    const userTextParts = [];
+    for (const userMessage of userMessages) {
+      const content = userMessage.content;
+      if (typeof content === "string") {
+        userSegments.push({ type: "text", value: content });
+        userTextParts.push(content);
+        continue;
+      }
+      for (const rawSegment of content) {
+        if (!isJsonObject(rawSegment)) {
+          continue;
+        }
+        const segmentType = asString(rawSegment.type);
+        if (segmentType === "file") {
+          const rawValue = asString(rawSegment.value);
+          if (!rawValue) {
+            continue;
+          }
+          const { displayPath, resolvedPath, attempted } = await resolveFileReference(
+            rawValue,
+            searchRoots
+          );
+          if (!resolvedPath) {
+            const attempts = attempted.length ? ["  Tried:", ...attempted.map((candidate) => `    ${candidate}`)] : void 0;
+            logWarning(`File not found: ${displayPath}`, attempts);
+            continue;
+          }
+          try {
+            const fileContent = (await (0, import_promises2.readFile)(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
+            if (isGuidelineFile(displayPath)) {
+              guidelinePaths.push(import_node_path2.default.resolve(resolvedPath));
+              if (verbose) {
+                console.log(`  [Guideline] Found: ${displayPath}`);
+                console.log(`    Resolved to: ${resolvedPath}`);
+              }
+            } else {
+              userSegments.push({
+                type: "file",
+                path: displayPath,
+                text: fileContent
+              });
+              if (verbose) {
+                console.log(`  [File] Found: ${displayPath}`);
+                console.log(`    Resolved to: ${resolvedPath}`);
+              }
+            }
+          } catch (error) {
+            logWarning(`Could not read file ${resolvedPath}: ${error.message}`);
+          }
+          continue;
+        }
+        const clonedSegment = cloneJsonObject(rawSegment);
+        userSegments.push(clonedSegment);
+        const inlineValue = clonedSegment.value;
+        if (typeof inlineValue === "string") {
+          userTextParts.push(inlineValue);
+        }
+      }
+    }
+    const codeSnippets = extractCodeBlocks(userSegments);
+    const assistantContent = assistantMessages[0]?.content;
+    const expectedAssistantRaw = normalizeAssistantContent(assistantContent);
+    const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
+    const testCaseGrader = coerceGrader(testcase.grader) ?? globalGrader;
+    const testCase = {
+      id,
+      conversation_id: conversationId,
+      task: userTextPrompt,
+      user_segments: userSegments,
+      expected_assistant_raw: expectedAssistantRaw,
+      guideline_paths: guidelinePaths.map((guidelinePath) => import_node_path2.default.resolve(guidelinePath)),
+      code_snippets: codeSnippets,
+      outcome,
+      grader: testCaseGrader
+    };
+    if (verbose) {
+      console.log(`
+[Test Case: ${id}]`);
+      if (testCase.guideline_paths.length > 0) {
+        console.log(`  Guidelines used: ${testCase.guideline_paths.length}`);
+        for (const guidelinePath of testCase.guideline_paths) {
+          console.log(`    - ${guidelinePath}`);
+        }
+      } else {
+        console.log("  No guidelines found");
+      }
+    }
+    results.push(testCase);
+  }
+  return results;
+}
+async function buildPromptInputs(testCase) {
+  const guidelineContents = [];
+  for (const rawPath of testCase.guideline_paths) {
+    const absolutePath = import_node_path2.default.resolve(rawPath);
+    if (!await fileExists2(absolutePath)) {
+      logWarning(`Could not read guideline file ${absolutePath}: file does not exist`);
+      continue;
+    }
+    try {
+      const content = (await (0, import_promises2.readFile)(absolutePath, "utf8")).replace(/\r\n/g, "\n");
+      guidelineContents.push(`=== ${import_node_path2.default.basename(absolutePath)} ===
+${content}`);
+    } catch (error) {
+      logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
+    }
+  }
+  const requestParts = [];
+  for (const segment of testCase.user_segments) {
+    const typeValue = segment.type;
+    if (typeof typeValue === "string" && typeValue === "file") {
+      const pathValue = segment.path;
+      const textValue = segment.text;
+      const label = typeof pathValue === "string" ? pathValue : "file";
+      const body = typeof textValue === "string" ? textValue : "";
+      requestParts.push(`=== ${label} ===
+${body}`);
+      continue;
+    }
+    if (typeof typeValue === "string" && typeValue === "text") {
+      const value = segment.value;
+      if (typeof value === "string") {
+        requestParts.push(value);
+      }
+      continue;
+    }
+    const genericValue = segment.value;
+    if (typeof genericValue === "string") {
+      requestParts.push(genericValue);
+    }
+  }
+  if (testCase.code_snippets.length > 0) {
+    requestParts.push(testCase.code_snippets.join("\n"));
+  }
+  const request = requestParts.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  const guidelines = guidelineContents.map((part) => part.trim()).filter((part) => part.length > 0).join("\n\n");
+  return { request, guidelines };
+}
+async function fileExists2(absolutePath) {
+  try {
+    await (0, import_promises2.access)(absolutePath, import_node_fs2.constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function resolveToAbsolutePath(candidate) {
+  if (candidate instanceof URL) {
+    return (0, import_node_url.fileURLToPath)(candidate);
+  }
+  if (typeof candidate === "string") {
+    if (candidate.startsWith("file://")) {
+      return (0, import_node_url.fileURLToPath)(new URL(candidate));
+    }
+    return import_node_path2.default.resolve(candidate);
+  }
+  throw new TypeError("Unsupported repoRoot value. Expected string or URL.");
+}
+function asString(value) {
+  return typeof value === "string" ? value : void 0;
+}
+function cloneJsonObject(source) {
+  const entries = Object.entries(source).map(([key, value]) => [key, cloneJsonValue(value)]);
+  return Object.fromEntries(entries);
+}
+function cloneJsonValue(value) {
+  if (value === null) {
+    return null;
+  }
+  if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => cloneJsonValue(item));
+  }
+  return cloneJsonObject(value);
+}
+function normalizeAssistantContent(content) {
+  if (typeof content === "string") {
+    return content;
+  }
+  if (!content) {
+    return "";
+  }
+  const parts = [];
+  for (const entry of content) {
+    if (typeof entry === "string") {
+      parts.push(entry);
+      continue;
+    }
+    const textValue = asString(entry["text"]);
+    if (typeof textValue === "string") {
+      parts.push(textValue);
+      continue;
+    }
+    const valueValue = asString(entry["value"]);
+    if (typeof valueValue === "string") {
+      parts.push(valueValue);
+      continue;
+    }
+    parts.push(JSON.stringify(entry));
+  }
+  return parts.join(" ");
+}
+function coerceGrader(candidate) {
+  if (typeof candidate !== "string") {
+    return void 0;
+  }
+  if (isGraderKind(candidate)) {
+    return candidate;
+  }
+  logWarning(`Unknown grader '${candidate}', falling back to default`);
+  return void 0;
+}
+function logWarning(message, details) {
+  if (details && details.length > 0) {
+    const detailBlock = details.join("\n");
+    console.warn(`${ANSI_YELLOW}Warning: ${message}
+${detailBlock}${ANSI_RESET}`);
+  } else {
+    console.warn(`${ANSI_YELLOW}Warning: ${message}${ANSI_RESET}`);
+  }
+}
+
+// src/evaluation/providers/ax.ts
+var import_ax = require("@ax-llm/ax");
+var DEFAULT_SYSTEM_PROMPT = "You are a careful assistant. Follow all provided instructions and do not fabricate results.";
+function buildChatPrompt(request) {
+  if (request.chatPrompt) {
+    return request.chatPrompt;
+  }
+  const systemSegments = [];
+  if (request.guidelines && request.guidelines.trim().length > 0) {
+    systemSegments.push(`Guidelines:
+${request.guidelines.trim()}`);
+  }
+  const metadataSystemPrompt = typeof request.metadata?.systemPrompt === "string" ? request.metadata.systemPrompt : void 0;
+  if (metadataSystemPrompt && metadataSystemPrompt.trim().length > 0) {
+    systemSegments.push(metadataSystemPrompt.trim());
+  }
+  const systemContent = systemSegments.length > 0 ? systemSegments.join("\n\n") : DEFAULT_SYSTEM_PROMPT;
+  const userContent = request.prompt.trim();
+  const prompt = [
+    {
+      role: "system",
+      content: systemContent
+    },
+    {
+      role: "user",
+      content: userContent
+    }
+  ];
+  return prompt;
+}
+function extractModelConfig(request, defaults) {
+  const temperature = request.temperature ?? defaults.temperature;
+  const maxTokens = request.maxOutputTokens ?? defaults.maxOutputTokens;
+  const config = {};
+  if (temperature !== void 0) {
+    config.temperature = temperature;
+  }
+  if (maxTokens !== void 0) {
+    config.maxTokens = maxTokens;
+  }
+  return Object.keys(config).length > 0 ? config : void 0;
+}
+function mapResponse(response) {
+  const primary = response.results[0];
+  const text = typeof primary?.content === "string" ? primary.content : "";
+  const reasoning = primary?.thought ?? primary?.thoughtBlock?.data;
+  const usage = toJsonObject(response.modelUsage);
+  return {
+    text,
+    reasoning,
+    raw: response,
+    usage
+  };
+}
+function toJsonObject(value) {
+  if (!value || typeof value !== "object") {
+    return void 0;
+  }
+  try {
+    return JSON.parse(JSON.stringify(value));
+  } catch {
+    return void 0;
+  }
+}
+function ensureChatResponse(result) {
+  if (typeof ReadableStream !== "undefined" && result instanceof ReadableStream) {
+    throw new Error("Streaming responses are not supported for this provider");
+  }
+  if (!result || typeof result !== "object" || !("results" in result)) {
+    throw new Error("Unexpected response type from AxAI provider");
+  }
+  return result;
+}
+var AzureProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `azure:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.ai = import_ax.AxAI.create({
+      name: "azure-openai",
+      apiKey: config.apiKey,
+      resourceName: config.resourceName,
+      deploymentName: config.deploymentName,
+      version: config.version,
+      config: {
+        stream: false
+      }
+    });
+  }
+  id;
+  kind = "azure";
+  targetName;
+  ai;
+  defaults;
+  async invoke(request) {
+    const chatPrompt = buildChatPrompt(request);
+    const modelConfig = extractModelConfig(request, this.defaults);
+    const response = await this.ai.chat(
+      {
+        chatPrompt,
+        model: this.config.deploymentName,
+        ...modelConfig ? { modelConfig } : {}
+      },
+      request.signal ? { abortSignal: request.signal } : void 0
+    );
+    return mapResponse(ensureChatResponse(response));
+  }
+};
+var AnthropicProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `anthropic:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens,
+      thinkingBudget: config.thinkingBudget
+    };
+    this.ai = import_ax.AxAI.create({
+      name: "anthropic",
+      apiKey: config.apiKey
+    });
+  }
+  id;
+  kind = "anthropic";
+  targetName;
+  ai;
+  defaults;
+  async invoke(request) {
+    const chatPrompt = buildChatPrompt(request);
+    const modelConfig = extractModelConfig(request, this.defaults);
+    const response = await this.ai.chat(
+      {
+        chatPrompt,
+        model: this.config.model,
+        ...modelConfig ? { modelConfig } : {}
+      },
+      request.signal ? { abortSignal: request.signal } : void 0
+    );
+    return mapResponse(ensureChatResponse(response));
+  }
+};
+var GeminiProvider = class {
+  constructor(targetName, config) {
+    this.config = config;
+    this.id = `gemini:${targetName}`;
+    this.targetName = targetName;
+    this.defaults = {
+      temperature: config.temperature,
+      maxOutputTokens: config.maxOutputTokens
+    };
+    this.ai = import_ax.AxAI.create({
+      name: "google-gemini",
+      apiKey: config.apiKey
+    });
+  }
+  id;
+  kind = "gemini";
+  targetName;
+  ai;
+  defaults;
+  async invoke(request) {
+    const chatPrompt = buildChatPrompt(request);
+    const modelConfig = extractModelConfig(request, this.defaults);
+    const response = await this.ai.chat(
+      {
+        chatPrompt,
+        model: this.config.model,
+        ...modelConfig ? { modelConfig } : {}
+      },
+      request.signal ? { abortSignal: request.signal } : void 0
+    );
+    return mapResponse(ensureChatResponse(response));
+  }
+};
+
+// src/evaluation/providers/mock.ts
+var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
+var MockProvider = class {
+  id;
+  kind = "mock";
+  targetName;
+  cannedResponse;
+  delayMs;
+  delayMinMs;
+  delayMaxMs;
+  constructor(targetName, config) {
+    this.id = `mock:${targetName}`;
+    this.targetName = targetName;
+    this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
+    this.delayMs = config.delayMs ?? 0;
+    this.delayMinMs = config.delayMinMs ?? 0;
+    this.delayMaxMs = config.delayMaxMs ?? 0;
+  }
+  async invoke(request) {
+    const delay = this.calculateDelay();
+    if (delay > 0) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+    return {
+      text: this.cannedResponse,
+      raw: {
+        prompt: request.prompt,
+        guidelines: request.guidelines
+      }
+    };
+  }
+  calculateDelay() {
+    if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
+      const min = Math.max(0, this.delayMinMs);
+      const max = Math.max(min, this.delayMaxMs);
+      return Math.floor(Math.random() * (max - min + 1)) + min;
+    }
+    return this.delayMs;
+  }
+};
+
+// src/evaluation/providers/targets.ts
+var import_zod = require("zod");
+var BASE_TARGET_SCHEMA = import_zod.z.object({
+  name: import_zod.z.string().min(1, "target name is required"),
+  provider: import_zod.z.string().min(1, "provider is required"),
+  settings: import_zod.z.record(import_zod.z.unknown()).optional(),
+  judge_target: import_zod.z.string().optional(),
+  workers: import_zod.z.number().int().min(1).optional()
+});
+var DEFAULT_AZURE_API_VERSION = "2024-10-01-preview";
+function normalizeAzureApiVersion(value) {
+  if (!value) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const trimmed = value.trim();
+  if (trimmed.length === 0) {
+    return DEFAULT_AZURE_API_VERSION;
+  }
+  const withoutPrefix = trimmed.replace(/^api[-_]?version\s*=\s*/i, "").trim();
+  return withoutPrefix.length > 0 ? withoutPrefix : DEFAULT_AZURE_API_VERSION;
+}
+function resolveTargetDefinition(definition, env = process.env) {
+  const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  const provider = parsed.provider.toLowerCase();
+  switch (provider) {
+    case "azure":
+    case "azure-openai":
+      return {
+        kind: "azure",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        config: resolveAzureConfig(parsed, env)
+      };
+    case "anthropic":
+      return {
+        kind: "anthropic",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        config: resolveAnthropicConfig(parsed, env)
+      };
+    case "gemini":
+    case "google":
+    case "google-gemini":
+      return {
+        kind: "gemini",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        config: resolveGeminiConfig(parsed, env)
+      };
+    case "mock":
+      return {
+        kind: "mock",
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        config: resolveMockConfig(parsed)
+      };
+    case "vscode":
+    case "vscode-insiders":
+      return {
+        kind: provider,
+        name: parsed.name,
+        judgeTarget: parsed.judge_target,
+        workers: parsed.workers,
+        config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
+      };
+    default:
+      throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
+  }
+}
+function resolveAzureConfig(target, env) {
+  const settings = target.settings ?? {};
+  const endpointSource = settings.endpoint ?? settings.resource ?? settings.resourceName;
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const deploymentSource = settings.deployment ?? settings.deploymentName ?? settings.model;
+  const versionSource = settings.version ?? settings.api_version;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const resourceName = resolveString(endpointSource, env, `${target.name} endpoint`);
+  const apiKey = resolveString(apiKeySource, env, `${target.name} api key`);
+  const deploymentName = resolveString(deploymentSource, env, `${target.name} deployment`);
+  const version = normalizeAzureApiVersion(
+    resolveOptionalString(versionSource, env, `${target.name} api version`)
+  );
+  const temperature = resolveOptionalNumber(temperatureSource, `${target.name} temperature`);
+  const maxOutputTokens = resolveOptionalNumber(
+    maxTokensSource,
+    `${target.name} max output tokens`
+  );
+  return {
+    resourceName,
+    deploymentName,
+    apiKey,
+    version,
+    temperature,
+    maxOutputTokens
+  };
+}
+function resolveAnthropicConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const thinkingBudgetSource = settings.thinking_budget ?? settings.thinkingBudget;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Anthropic api key`);
+  const model = resolveString(modelSource, env, `${target.name} Anthropic model`);
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`),
+    thinkingBudget: resolveOptionalNumber(thinkingBudgetSource, `${target.name} thinking budget`)
+  };
+}
+function resolveGeminiConfig(target, env) {
+  const settings = target.settings ?? {};
+  const apiKeySource = settings.api_key ?? settings.apiKey;
+  const modelSource = settings.model ?? settings.deployment ?? settings.variant;
+  const temperatureSource = settings.temperature;
+  const maxTokensSource = settings.max_output_tokens ?? settings.maxTokens;
+  const apiKey = resolveString(apiKeySource, env, `${target.name} Google API key`);
+  const model = resolveOptionalString(modelSource, env, `${target.name} Gemini model`, {
+    allowLiteral: true,
+    optionalEnv: true
+  }) ?? "gemini-2.5-flash";
+  return {
+    apiKey,
+    model,
+    temperature: resolveOptionalNumber(temperatureSource, `${target.name} temperature`),
+    maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
+  };
+}
+function resolveMockConfig(target) {
+  const settings = target.settings ?? {};
+  const response = typeof settings.response === "string" ? settings.response : void 0;
+  return { response };
+}
+function resolveVSCodeConfig(target, env, insiders) {
+  const settings = target.settings ?? {};
+  const workspaceTemplateEnvVar = resolveOptionalLiteralString(settings.workspace_template ?? settings.workspaceTemplate);
+  const workspaceTemplate = workspaceTemplateEnvVar ? resolveOptionalString(workspaceTemplateEnvVar, env, `${target.name} workspace template path`, {
+    allowLiteral: false,
+    optionalEnv: true
+  }) : void 0;
+  const commandSource = settings.vscode_cmd ?? settings.command;
+  const waitSource = settings.wait;
+  const dryRunSource = settings.dry_run ?? settings.dryRun;
+  const subagentRootSource = settings.subagent_root ?? settings.subagentRoot;
+  const defaultCommand = insiders ? "code-insiders" : "code";
+  const command = resolveOptionalLiteralString(commandSource) ?? defaultCommand;
+  return {
+    command,
+    waitForResponse: resolveOptionalBoolean(waitSource) ?? true,
+    dryRun: resolveOptionalBoolean(dryRunSource) ?? false,
+    subagentRoot: resolveOptionalString(subagentRootSource, env, `${target.name} subagent root`, {
+      allowLiteral: true,
+      optionalEnv: true
+    }),
+    workspaceTemplate
+  };
+}
+function resolveString(source, env, description, allowLiteral = false) {
+  const value = resolveOptionalString(source, env, description, {
+    allowLiteral,
+    optionalEnv: false
+  });
+  if (value === void 0) {
+    throw new Error(`${description} is required`);
+  }
+  return value;
+}
+function resolveOptionalString(source, env, description, options) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "string") {
+    throw new Error(`${description} must be a string`);
+  }
+  const trimmed = source.trim();
+  if (trimmed.length === 0) {
+    return void 0;
+  }
+  const envValue = env[trimmed];
+  if (envValue !== void 0) {
+    if (envValue.trim().length === 0) {
+      throw new Error(`Environment variable '${trimmed}' for ${description} is empty`);
+    }
+    return envValue;
+  }
+  const allowLiteral = options?.allowLiteral ?? false;
+  const optionalEnv = options?.optionalEnv ?? false;
+  if (!allowLiteral && isLikelyEnvReference(trimmed)) {
+    if (optionalEnv) {
+      return void 0;
+    }
+    throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
+  }
+  return trimmed;
+}
+function resolveOptionalLiteralString(source) {
+  if (source === void 0 || source === null) {
+    return void 0;
+  }
+  if (typeof source !== "string") {
+    throw new Error("expected string value");
+  }
+  const trimmed = source.trim();
+  return trimmed.length > 0 ? trimmed : void 0;
+}
+function resolveOptionalNumber(source, description) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
+  }
+  if (typeof source === "number") {
+    return Number.isFinite(source) ? source : void 0;
+  }
+  if (typeof source === "string") {
+    const numeric = Number(source);
+    if (Number.isFinite(numeric)) {
+      return numeric;
+    }
+  }
+  throw new Error(`${description} must be a number`);
+}
+function resolveOptionalBoolean(source) {
+  if (source === void 0 || source === null || source === "") {
+    return void 0;
+  }
+  if (typeof source === "boolean") {
+    return source;
+  }
+  if (typeof source === "string") {
+    const lowered = source.trim().toLowerCase();
+    if (lowered === "true" || lowered === "1") {
+      return true;
+    }
+    if (lowered === "false" || lowered === "0") {
+      return false;
+    }
+  }
+  throw new Error("expected boolean value");
+}
+function isLikelyEnvReference(value) {
+  return /^[A-Z0-9_]+$/.test(value);
+}
+
+// src/evaluation/providers/vscode.ts
+var import_promises3 = require("fs/promises");
+var import_node_os = require("os");
+var import_node_path3 = __toESM(require("path"), 1);
+var import_subagent = require("subagent");
+var PROMPT_FILE_PREFIX = "bbeval-vscode-";
+var VSCodeProvider = class {
+  id;
+  kind;
+  targetName;
+  config;
+  constructor(targetName, config, kind) {
+    this.id = `${kind}:${targetName}`;
+    this.kind = kind;
+    this.targetName = targetName;
+    this.config = config;
+  }
+  async invoke(request) {
+    if (request.signal?.aborted) {
+      throw new Error("VS Code provider request was aborted before dispatch");
+    }
+    const attachments = normalizeAttachments(request.attachments);
+    const promptContent = buildPromptDocument(request, attachments);
+    const directory = await (0, import_promises3.mkdtemp)(import_node_path3.default.join((0, import_node_os.tmpdir)(), PROMPT_FILE_PREFIX));
+    const promptPath = import_node_path3.default.join(directory, `${request.testCaseId ?? "request"}.prompt.md`);
+    try {
+      await (0, import_promises3.writeFile)(promptPath, promptContent, "utf8");
+      const session = await (0, import_subagent.dispatchAgentSession)({
+        userQuery: composeUserQuery(request),
+        promptFile: promptPath,
+        extraAttachments: attachments,
+        wait: this.config.waitForResponse,
+        dryRun: this.config.dryRun,
+        vscodeCmd: this.config.command,
+        subagentRoot: this.config.subagentRoot,
+        workspaceTemplate: this.config.workspaceTemplate,
+        silent: true
+      });
+      if (session.exitCode !== 0 || !session.responseFile) {
+        const failure = session.error ?? "VS Code subagent did not produce a response";
+        throw new Error(failure);
+      }
+      if (this.config.dryRun) {
+        return {
+          text: "",
+          raw: {
+            session,
+            promptFile: promptPath,
+            attachments
+          }
+        };
+      }
+      const responseText = await (0, import_promises3.readFile)(session.responseFile, "utf8");
+      return {
+        text: responseText,
+        raw: {
+          session,
+          promptFile: promptPath,
+          attachments
+        }
+      };
+    } finally {
+      await (0, import_promises3.rm)(directory, { recursive: true, force: true });
+    }
+  }
+};
+function buildPromptDocument(request, attachments) {
+  const parts = [];
+  const instructionFiles = collectInstructionFiles(attachments);
+  if (instructionFiles.length > 0) {
+    parts.push(buildMandatoryPrereadBlock(instructionFiles));
+  }
+  parts.push(`# BbEval Request`);
+  if (request.testCaseId) {
+    parts.push(`- Test Case: ${request.testCaseId}`);
+  }
+  if (request.metadata?.target) {
+    parts.push(`- Target: ${String(request.metadata.target)}`);
+  }
+  parts.push("\n## Task\n", request.prompt.trim());
+  if (request.guidelines && request.guidelines.trim().length > 0) {
+    parts.push("\n## Guidelines\n", request.guidelines.trim());
+  }
+  if (attachments && attachments.length > 0) {
+    const attachmentList = attachments.map((item) => `- ${item}`).join("\n");
+    parts.push("\n## Attachments\n", attachmentList);
+  }
+  return parts.join("\n").trim();
+}
+function buildMandatoryPrereadBlock(instructionFiles) {
+  if (instructionFiles.length === 0) {
+    return "";
+  }
+  const fileList = [];
+  const tokenList = [];
+  let counter = 0;
+  for (const absolutePath of instructionFiles) {
+    counter += 1;
+    const fileName = import_node_path3.default.basename(absolutePath);
+    const fileUri = pathToFileUri(absolutePath);
+    fileList.push(`[${fileName}](${fileUri})`);
+    tokenList.push(`INSTRUCTIONS_READ: \`${fileName}\` i=${counter} SHA256=<hex>`);
+  }
+  const filesText = fileList.join(", ");
+  const tokensText = tokenList.join("\n");
+  const instruction = [
+    `Read all instruction files: ${filesText}.`,
+    `After reading each file, compute its SHA256 hash using this PowerShell command:`,
+    "`Get-FileHash -Algorithm SHA256 -LiteralPath '<file-path>' | Select-Object -ExpandProperty Hash`.",
+    `Then include, at the top of your reply, these exact tokens on separate lines:
+`,
+    tokensText,
+    `
+Replace \`<hex>\` with the actual SHA256 hash value computed from the PowerShell command.`,
+    `If any file is missing, fail with ERROR: missing-file <filename> and stop.
+`,
+    `Then fetch all documentation required by the instructions before proceeding with your task.`
+  ].join(" ");
+  return `[[ ## mandatory_pre_read ## ]]
+
+${instruction}
+
+`;
+}
+function collectInstructionFiles(attachments) {
+  if (!attachments || attachments.length === 0) {
+    return [];
+  }
+  const unique = /* @__PURE__ */ new Map();
+  for (const attachment of attachments) {
+    if (!isInstructionPath(attachment)) {
+      continue;
+    }
+    const absolutePath = import_node_path3.default.resolve(attachment);
+    if (!unique.has(absolutePath)) {
+      unique.set(absolutePath, absolutePath);
+    }
+  }
+  return Array.from(unique.values());
+}
+function isInstructionPath(filePath) {
+  const normalized = filePath.split(import_node_path3.default.sep).join("/");
+  return normalized.endsWith(".instructions.md") || normalized.includes("/instructions/") || normalized.endsWith(".prompt.md") || normalized.includes("/prompts/");
+}
+function pathToFileUri(filePath) {
+  const absolutePath = import_node_path3.default.isAbsolute(filePath) ? filePath : import_node_path3.default.resolve(filePath);
+  const normalizedPath = absolutePath.replace(/\\/g, "/");
+  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
+    return `file:///${normalizedPath}`;
+  }
+  return `file://${normalizedPath}`;
+}
+function composeUserQuery(request) {
+  const segments = [];
+  segments.push(request.prompt.trim());
+  if (request.guidelines && request.guidelines.trim().length > 0) {
+    segments.push("\nGuidelines:\n", request.guidelines.trim());
+  }
+  return segments.join("\n").trim();
+}
+function normalizeAttachments(attachments) {
+  if (!attachments || attachments.length === 0) {
+    return void 0;
+  }
+  const deduped = /* @__PURE__ */ new Set();
+  for (const attachment of attachments) {
+    deduped.add(import_node_path3.default.resolve(attachment));
+  }
+  return Array.from(deduped);
+}
+async function ensureVSCodeSubagents(options) {
+  const { kind, count, verbose = false } = options;
+  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
+  const subagentRoot = (0, import_subagent.getSubagentRoot)(vscodeCmd);
+  try {
+    if (verbose) {
+      console.log(`Provisioning ${count} subagent(s) via: subagent ${vscodeCmd} provision`);
+    }
+    const result = await (0, import_subagent.provisionSubagents)({
+      targetRoot: subagentRoot,
+      subagents: count,
+      dryRun: false
+    });
+    if (verbose) {
+      if (result.created.length > 0) {
+        console.log(`Created ${result.created.length} new subagent(s)`);
+      }
+      if (result.skippedExisting.length > 0) {
+        console.log(`Reusing ${result.skippedExisting.length} existing unlocked subagent(s)`);
+      }
+      console.log(`
+total unlocked subagents available: ${result.created.length + result.skippedExisting.length}`);
+    }
+    return {
+      provisioned: true,
+      message: `Provisioned ${count} subagent(s): ${result.created.length} created, ${result.skippedExisting.length} reused`
+    };
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    if (verbose) {
+      console.warn(`Provisioning failed (continuing anyway): ${errorMessage}`);
+    }
+    return {
+      provisioned: false,
+      message: `Provisioning failed: ${errorMessage}`
+    };
+  }
+}
+
+// src/evaluation/providers/targets-file.ts
+var import_node_fs3 = require("fs");
+var import_promises4 = require("fs/promises");
+var import_node_path4 = __toESM(require("path"), 1);
+var import_yaml2 = require("yaml");
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function checkVersion(parsed, absolutePath) {
+  const version = typeof parsed.version === "number" ? parsed.version : typeof parsed.version === "string" ? parseFloat(parsed.version) : void 0;
+  if (version === void 0) {
+    throw new Error(
+      `Missing version field in targets.yaml at ${absolutePath}.
+Please add 'version: 2.0' at the top of the file.`
+    );
+  }
+  if (version < 2) {
+    throw new Error(
+      `Outdated targets.yaml format (version ${version}) at ${absolutePath}.
+Please update to version 2.0 format with 'targets' array.`
+    );
+  }
+}
+function extractTargetsArray(parsed, absolutePath) {
+  const targets = parsed.targets;
+  if (!Array.isArray(targets)) {
+    throw new Error(`targets.yaml at ${absolutePath} must have a 'targets' array`);
+  }
+  return targets;
+}
+function assertTargetDefinition(value, index, filePath) {
+  if (!isRecord(value)) {
+    throw new Error(`targets.yaml entry at index ${index} in ${filePath} must be an object`);
+  }
+  const name = value.name;
+  const provider = value.provider;
+  const settings = value.settings;
+  const judgeTarget = value.judge_target;
+  if (typeof name !== "string" || name.trim().length === 0) {
+    throw new Error(`targets.yaml entry at index ${index} in ${filePath} is missing a valid 'name'`);
+  }
+  if (typeof provider !== "string" || provider.trim().length === 0) {
+    throw new Error(`targets.yaml entry '${name}' in ${filePath} is missing a valid 'provider'`);
+  }
+  return {
+    name,
+    provider,
+    settings: isRecord(settings) ? settings : void 0,
+    judge_target: typeof judgeTarget === "string" ? judgeTarget : void 0
+  };
+}
+async function fileExists3(filePath) {
+  try {
+    await (0, import_promises4.access)(filePath, import_node_fs3.constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function readTargetDefinitions(filePath) {
+  const absolutePath = import_node_path4.default.resolve(filePath);
+  if (!await fileExists3(absolutePath)) {
+    throw new Error(`targets.yaml not found at ${absolutePath}`);
+  }
+  const raw = await (0, import_promises4.readFile)(absolutePath, "utf8");
+  const parsed = (0, import_yaml2.parse)(raw);
+  if (!isRecord(parsed)) {
+    throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with 'version' and 'targets' fields`);
+  }
+  checkVersion(parsed, absolutePath);
+  const targets = extractTargetsArray(parsed, absolutePath);
+  const definitions = targets.map((entry, index) => assertTargetDefinition(entry, index, absolutePath));
+  return definitions;
+}
+function listTargetNames(definitions) {
+  return definitions.map((definition) => definition.name);
+}
+
+// src/evaluation/providers/index.ts
+function createProvider(target) {
+  switch (target.kind) {
+    case "azure":
+      return new AzureProvider(target.name, target.config);
+    case "anthropic":
+      return new AnthropicProvider(target.name, target.config);
+    case "gemini":
+      return new GeminiProvider(target.name, target.config);
+    case "mock":
+      return new MockProvider(target.name, target.config);
+    case "vscode":
+    case "vscode-insiders":
+      return new VSCodeProvider(target.name, target.config, target.kind);
+    default: {
+      const neverTarget = target;
+      throw new Error(`Unsupported provider kind ${neverTarget.kind}`);
+    }
+  }
+}
+function resolveAndCreateProvider(definition, env = process.env) {
+  const resolved = resolveTargetDefinition(definition, env);
+  return createProvider(resolved);
+}
+
+// src/evaluation/scoring.ts
+var KEY_TERM_MATCH_THRESHOLD = 0.5;
+var ACTION_WORDS = /* @__PURE__ */ new Set([
+  "use",
+  "avoid",
+  "prefer",
+  "replace",
+  "consider",
+  "ensure",
+  "remove",
+  "add"
+]);
+var STOP_WORDS = /* @__PURE__ */ new Set([
+  "the",
+  "a",
+  "an",
+  "and",
+  "or",
+  "but",
+  "in",
+  "on",
+  "at",
+  "to",
+  "for",
+  "of",
+  "with",
+  "by",
+  "is",
+  "are",
+  "was",
+  "were",
+  "be",
+  "been",
+  "being",
+  "have",
+  "has",
+  "had",
+  "do",
+  "does",
+  "did",
+  "will",
+  "would",
+  "could",
+  "should"
+]);
+var ERROR_PREFIXES = [
+  "error:",
+  "err:",
+  "vs code command failed",
+  "exception",
+  "traceback",
+  "no response file was generated",
+  "timed out",
+  "cli not found"
+];
+function extractAspects(expectedResponse) {
+  const lines = expectedResponse.split(/\r?\n/).map((line) => line.trim());
+  const aspects = [];
+  for (const line of lines) {
+    if (line.length === 0) {
+      continue;
+    }
+    const bulletMatch = /^([-*•]|[0-9]+\.)\s*(.+)$/.exec(line);
+    if (bulletMatch) {
+      const normalized = normalizeAspect(bulletMatch[2]);
+      if (normalized.length > 0) {
+        aspects.push(normalized);
+      }
+      continue;
+    }
+    const lowered = line.toLowerCase();
+    if (Array.from(ACTION_WORDS).some((word) => lowered.startsWith(word))) {
+      const normalized = normalizeAspect(line);
+      if (normalized.length > 0) {
+        aspects.push(normalized);
+      }
+    }
+  }
+  return aspects;
+}
+function calculateHits(candidateResponse, expectedAspects) {
+  const { normalizedText, words } = normalizeCandidate(candidateResponse);
+  const hits = [];
+  for (const aspect of expectedAspects) {
+    if (matchesAspect(aspect, normalizedText, words)) {
+      hits.push(aspect);
+    }
+  }
+  return hits;
+}
+function calculateMisses(candidateResponse, expectedAspects, resolvedHits) {
+  const hits = new Set(resolvedHits ?? calculateHits(candidateResponse, expectedAspects));
+  return expectedAspects.filter((aspect) => !hits.has(aspect));
+}
+function scoreCandidateResponse(candidateResponse, expectedAspects) {
+  if (expectedAspects.length === 0) {
+    if (isErrorLike(candidateResponse)) {
+      return {
+        score: 0,
+        hits: [],
+        misses: ["Model produced an error instead of an answer."],
+        hitCount: 0,
+        totalAspects: 0,
+        rawAspects: []
+      };
+    }
+    return {
+      score: 1,
+      hits: [],
+      misses: [],
+      hitCount: 0,
+      totalAspects: 0,
+      rawAspects: []
+    };
+  }
+  const hits = calculateHits(candidateResponse, expectedAspects);
+  const misses = expectedAspects.filter((aspect) => !hits.includes(aspect));
+  const score = expectedAspects.length > 0 ? hits.length / expectedAspects.length : 0;
+  return {
+    score,
+    hits,
+    misses,
+    hitCount: hits.length,
+    totalAspects: expectedAspects.length,
+    rawAspects: expectedAspects
+  };
+}
+function isErrorLike(text) {
+  if (!text) {
+    return false;
+  }
+  const lowered = text.trim().toLowerCase();
+  return ERROR_PREFIXES.some((prefix) => lowered.startsWith(prefix));
+}
+function normalizeAspect(aspect) {
+  const sanitized = aspect.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim();
+  return sanitized;
+}
+function normalizeCandidate(candidate) {
+  const lowered = candidate.toLowerCase();
+  const normalizedText = lowered.replace(/[^\w\s]/g, " ");
+  const words = new Set(normalizedText.split(/\s+/).filter((word) => word.length > 0));
+  return { normalizedText, words };
+}
+function matchesAspect(aspect, candidateNormalized, candidateWords) {
+  const keyTerms = extractKeyTerms(aspect);
+  if (keyTerms.length === 0) {
+    return false;
+  }
+  const matches = keyTerms.filter((term) => candidateWords.has(term)).length;
+  const ratio = matches / keyTerms.length;
+  if (ratio >= KEY_TERM_MATCH_THRESHOLD) {
+    return true;
+  }
+  const aspectWords = aspect.split(" ");
+  if (aspectWords.length >= 2) {
+    for (let index = 0; index < aspectWords.length - 1; index += 1) {
+      const phrase = `${aspectWords[index]} ${aspectWords[index + 1]}`;
+      if (candidateNormalized.includes(phrase)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+function extractKeyTerms(aspect, maxTerms = 5) {
+  const terms = [];
+  const words = aspect.split(" ");
+  for (const word of words) {
+    if (word.length <= 2) {
+      continue;
+    }
+    if (STOP_WORDS.has(word)) {
+      continue;
+    }
+    terms.push(word);
+    if (terms.length >= maxTerms) {
+      break;
+    }
+  }
+  return terms;
+}
+
+// src/evaluation/grading.ts
+var import_node_crypto = require("crypto");
+var HeuristicGrader = class {
+  kind = "heuristic";
+  grade(context) {
+    const expectedAspects = extractAspects(context.testCase.expected_assistant_raw);
+    const result = scoreCandidateResponse(context.candidate, expectedAspects);
+    const misses = [...result.misses];
+    if (expectedAspects.length === 0 && isErrorLike(context.candidate)) {
+      const firstLine = context.candidate.split(/\r?\n/)[0]?.trim();
+      if (firstLine && !misses.includes(firstLine)) {
+        misses.unshift(firstLine);
+      }
+    }
+    return {
+      score: result.score,
+      hits: result.hits,
+      misses,
+      expectedAspectCount: result.totalAspects,
+      rawAspects: result.rawAspects
+    };
+  }
+};
+var QualityGrader = class {
+  kind = "llm_judge";
+  resolveJudgeProvider;
+  maxOutputTokens;
+  temperature;
+  constructor(options) {
+    this.resolveJudgeProvider = options.resolveJudgeProvider;
+    this.maxOutputTokens = options.maxOutputTokens;
+    this.temperature = options.temperature;
+  }
+  async grade(context) {
+    const judgeProvider = await this.resolveJudgeProvider(context);
+    if (!judgeProvider) {
+      throw new Error("No judge provider available for LLM grading");
+    }
+    const prompt = buildQualityPrompt(context.testCase, context.candidate);
+    const metadata = {
+      systemPrompt: QUALITY_SYSTEM_PROMPT
+    };
+    const response = await judgeProvider.invoke({
+      prompt,
+      metadata,
+      testCaseId: context.testCase.id,
+      attempt: context.attempt,
+      maxOutputTokens: this.maxOutputTokens,
+      temperature: this.temperature
+    });
+    const parsed = parseQualityResponse(response);
+    const score = clampScore(parsed.score ?? 0);
+    const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
+    const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
+    const reasoning = parsed.reasoning ?? response.reasoning;
+    const graderRawRequest = {
+      id: (0, import_node_crypto.randomUUID)(),
+      provider: judgeProvider.id,
+      prompt,
+      systemPrompt: QUALITY_SYSTEM_PROMPT,
+      target: context.target.name
+    };
+    return {
+      score,
+      hits,
+      misses,
+      expectedAspectCount: hits.length + misses.length || 1,
+      reasoning,
+      graderRawRequest
+    };
+  }
+};
+var QUALITY_SYSTEM_PROMPT = [
+  "You are an expert evaluator. Your goal is to grade the generated_answer based on how well it achieves the expected_outcome for the original task.",
+  "",
+  "Use the reference_answer as a gold standard for a high-quality response. The generated_answer does not need to match it verbatim, but it should capture the key points and follow the same spirit.",
+  "",
+  "Be concise and focused in your evaluation. Provide succinct, specific feedback rather than verbose explanations.",
+  "",
+  "You must respond with a single JSON object matching this schema:",
+  "",
+  "{",
+  '  "score": <number between 0.0 and 1.0>,',
+  '  "hits": [<array of strings, max 4 items, brief specific achievements>],',
+  '  "misses": [<array of strings, max 4 items, brief specific failures or omissions, empty if none>],',
+  '  "reasoning": "<string, concise explanation for the score, 1-2 sentences max>"',
+  "}"
+].join("\n");
+function buildQualityPrompt(testCase, candidate) {
+  const parts = [
+    "[[ ## expected_outcome ## ]]",
+    testCase.outcome,
+    "",
+    "[[ ## request ## ]]",
+    testCase.task,
+    "",
+    "[[ ## reference_answer ## ]]",
+    testCase.expected_assistant_raw,
+    "",
+    "[[ ## generated_answer ## ]]",
+    candidate,
+    "",
+    "Respond with a single JSON object matching the schema described in the system prompt."
+  ];
+  return parts.join("\n");
+}
+function clampScore(value) {
+  if (Number.isNaN(value) || !Number.isFinite(value)) {
+    return 0;
+  }
+  if (value < 0) {
+    return 0;
+  }
+  if (value > 1) {
+    return 1;
+  }
+  return value;
+}
+function parseQualityResponse(response) {
+  const text = typeof response.text === "string" ? response.text.trim() : "";
+  if (text.length === 0) {
+    return {};
+  }
+  const direct = attemptParseJson(text);
+  if (direct && validateQualityJson(direct)) {
+    return direct;
+  }
+  const extracted = extractJsonBlob(text);
+  if (extracted) {
+    const parsed = attemptParseJson(extracted);
+    if (parsed && validateQualityJson(parsed)) {
+      return parsed;
+    }
+  }
+  return {};
+}
+function attemptParseJson(text) {
+  try {
+    const parsed = JSON.parse(text);
+    const score = typeof parsed.score === "number" ? parsed.score : void 0;
+    const hits = parsed.hits;
+    const misses = parsed.misses;
+    const reasoning = typeof parsed.reasoning === "string" ? parsed.reasoning : void 0;
+    return { score, hits, misses, reasoning };
+  } catch {
+    return void 0;
+  }
+}
+function validateQualityJson(parsed) {
+  if (typeof parsed.score !== "number") {
+    return false;
+  }
+  if (Number.isNaN(parsed.score) || !Number.isFinite(parsed.score)) {
+    return false;
+  }
+  if (parsed.score < 0 || parsed.score > 1) {
+    return false;
+  }
+  if (parsed.hits !== void 0) {
+    if (!Array.isArray(parsed.hits)) {
+      return false;
+    }
+    if (!parsed.hits.every((item) => typeof item === "string")) {
+      return false;
+    }
+  }
+  if (parsed.misses !== void 0) {
+    if (!Array.isArray(parsed.misses)) {
+      return false;
+    }
+    if (!parsed.misses.every((item) => typeof item === "string")) {
+      return false;
+    }
+  }
+  if (parsed.reasoning !== void 0 && typeof parsed.reasoning !== "string") {
+    return false;
+  }
+  return true;
+}
+function extractJsonBlob(text) {
+  const match = text.match(/\{[\s\S]*\}/);
+  return match?.[0];
+}
+function isNonEmptyString(value) {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+// src/evaluation/orchestrator.ts
+var import_node_crypto2 = require("crypto");
+var import_promises5 = require("fs/promises");
+var import_node_path5 = __toESM(require("path"), 1);
+
+// ../../node_modules/.pnpm/yocto-queue@1.2.1/node_modules/yocto-queue/index.js
+var Node = class {
+  value;
+  next;
+  constructor(value) {
+    this.value = value;
+  }
+};
+var Queue = class {
+  #head;
+  #tail;
+  #size;
+  constructor() {
+    this.clear();
+  }
+  enqueue(value) {
+    const node = new Node(value);
+    if (this.#head) {
+      this.#tail.next = node;
+      this.#tail = node;
+    } else {
+      this.#head = node;
+      this.#tail = node;
+    }
+    this.#size++;
+  }
+  dequeue() {
+    const current = this.#head;
+    if (!current) {
+      return;
+    }
+    this.#head = this.#head.next;
+    this.#size--;
+    return current.value;
+  }
+  peek() {
+    if (!this.#head) {
+      return;
+    }
+    return this.#head.value;
+  }
+  clear() {
+    this.#head = void 0;
+    this.#tail = void 0;
+    this.#size = 0;
+  }
+  get size() {
+    return this.#size;
+  }
+  *[Symbol.iterator]() {
+    let current = this.#head;
+    while (current) {
+      yield current.value;
+      current = current.next;
+    }
+  }
+  *drain() {
+    while (this.#head) {
+      yield this.dequeue();
+    }
+  }
+};
+
+// ../../node_modules/.pnpm/p-limit@6.2.0/node_modules/p-limit/index.js
+function pLimit(concurrency) {
+  validateConcurrency(concurrency);
+  const queue = new Queue();
+  let activeCount = 0;
+  const resumeNext = () => {
+    if (activeCount < concurrency && queue.size > 0) {
+      queue.dequeue()();
+      activeCount++;
+    }
+  };
+  const next = () => {
+    activeCount--;
+    resumeNext();
+  };
+  const run = async (function_, resolve, arguments_) => {
+    const result = (async () => function_(...arguments_))();
+    resolve(result);
+    try {
+      await result;
+    } catch {
+    }
+    next();
+  };
+  const enqueue = (function_, resolve, arguments_) => {
+    new Promise((internalResolve) => {
+      queue.enqueue(internalResolve);
+    }).then(
+      run.bind(void 0, function_, resolve, arguments_)
+    );
+    (async () => {
+      await Promise.resolve();
+      if (activeCount < concurrency) {
+        resumeNext();
+      }
+    })();
+  };
+  const generator = (function_, ...arguments_) => new Promise((resolve) => {
+    enqueue(function_, resolve, arguments_);
+  });
+  Object.defineProperties(generator, {
+    activeCount: {
+      get: () => activeCount
+    },
+    pendingCount: {
+      get: () => queue.size
+    },
+    clearQueue: {
+      value() {
+        queue.clear();
+      }
+    },
+    concurrency: {
+      get: () => concurrency,
+      set(newConcurrency) {
+        validateConcurrency(newConcurrency);
+        concurrency = newConcurrency;
+        queueMicrotask(() => {
+          while (activeCount < concurrency && queue.size > 0) {
+            resumeNext();
+          }
+        });
+      }
+    }
+  });
+  return generator;
+}
+function validateConcurrency(concurrency) {
+  if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
+    throw new TypeError("Expected `concurrency` to be a number from 1 and up");
+  }
+}
+
+// src/evaluation/orchestrator.ts
+async function runEvaluation(options) {
+  const {
+    testFilePath,
+    repoRoot,
+    target,
+    targets,
+    env,
+    providerFactory,
+    graders,
+    maxRetries,
+    agentTimeoutMs,
+    promptDumpDir,
+    cache,
+    useCache,
+    now,
+    testId,
+    verbose,
+    onResult,
+    onProgress
+  } = options;
+  const load = loadTestCases;
+  const testCases = await load(testFilePath, repoRoot, { verbose });
+  const filteredTestCases = filterTestCases(testCases, testId);
+  if (filteredTestCases.length === 0) {
+    if (testId) {
+      throw new Error(`Test case with id '${testId}' not found in ${testFilePath}`);
+    }
+    return [];
+  }
+  const resolvedTargetsByName = /* @__PURE__ */ new Map();
+  resolvedTargetsByName.set(target.name, target);
+  const targetDefinitions = /* @__PURE__ */ new Map();
+  for (const definition of targets ?? []) {
+    targetDefinitions.set(definition.name, definition);
+  }
+  const envLookup = env ?? process.env;
+  const providerCache = /* @__PURE__ */ new Map();
+  const getOrCreateProvider = (resolved) => {
+    const existing = providerCache.get(resolved.name);
+    if (existing) {
+      return existing;
+    }
+    const factory = providerFactory ?? createProvider;
+    const instance = factory(resolved);
+    providerCache.set(resolved.name, instance);
+    return instance;
+  };
+  const resolveTargetByName = (name) => {
+    if (resolvedTargetsByName.has(name)) {
+      return resolvedTargetsByName.get(name);
+    }
+    const definition = targetDefinitions.get(name);
+    if (!definition) {
+      return void 0;
+    }
+    const resolved = resolveTargetDefinition(definition, envLookup);
+    resolvedTargetsByName.set(name, resolved);
+    return resolved;
+  };
+  const resolveJudgeProvider = async (targetContext) => {
+    const judgeName = targetContext.judgeTarget ?? targetContext.name;
+    const resolvedJudge = resolveTargetByName(judgeName);
+    if (!resolvedJudge) {
+      return getOrCreateProvider(targetContext);
+    }
+    return getOrCreateProvider(resolvedJudge);
+  };
+  const graderRegistry = buildGraderRegistry(graders, resolveJudgeProvider);
+  const primaryProvider = getOrCreateProvider(target);
+  if (onProgress && filteredTestCases.length > 0) {
+    for (let i = 0; i < filteredTestCases.length; i++) {
+      await onProgress({
+        workerId: i + 1,
+        testId: filteredTestCases[i].id,
+        status: "pending"
+      });
+    }
+  }
+  const workers = options.maxConcurrency ?? target.workers ?? 1;
+  const limit = pLimit(workers);
+  let nextWorkerId = 1;
+  const workerIdByTestId = /* @__PURE__ */ new Map();
+  const promises = filteredTestCases.map(
+    (testCase) => limit(async () => {
+      const workerId = nextWorkerId++;
+      workerIdByTestId.set(testCase.id, workerId);
+      if (onProgress) {
+        await onProgress({
+          workerId,
+          testId: testCase.id,
+          status: "running",
+          startedAt: Date.now()
+        });
+      }
+      try {
+        const judgeProvider = await resolveJudgeProvider(target);
+        const result = await runTestCase({
+          testCase,
+          provider: primaryProvider,
+          target,
+          graders: graderRegistry,
+          maxRetries,
+          agentTimeoutMs,
+          promptDumpDir,
+          cache,
+          useCache,
+          now,
+          judgeProvider
+        });
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: testCase.id,
+            status: "completed",
+            startedAt: 0,
+            // Not used for completed status
+            completedAt: Date.now()
+          });
+        }
+        if (onResult) {
+          await onResult(result);
+        }
+        return result;
+      } catch (error) {
+        if (onProgress) {
+          await onProgress({
+            workerId,
+            testId: testCase.id,
+            status: "failed",
+            completedAt: Date.now(),
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
+        throw error;
+      }
+    })
+  );
+  const settled = await Promise.allSettled(promises);
+  const results = [];
+  for (let i = 0; i < settled.length; i++) {
+    const outcome = settled[i];
+    if (outcome.status === "fulfilled") {
+      results.push(outcome.value);
+    } else {
+      const testCase = filteredTestCases[i];
+      const promptInputs = await buildPromptInputs(testCase);
+      const errorResult = buildErrorResult(
+        testCase,
+        target.name,
+        (now ?? (() => /* @__PURE__ */ new Date()))(),
+        outcome.reason,
+        promptInputs
+      );
+      results.push(errorResult);
+      if (onResult) {
+        await onResult(errorResult);
+      }
+    }
+  }
+  return results;
+}
+async function runTestCase(options) {
+  const {
+    testCase,
+    provider,
+    target,
+    graders,
+    now,
+    maxRetries,
+    agentTimeoutMs,
+    promptDumpDir,
+    cache,
+    useCache,
+    signal,
+    judgeProvider
+  } = options;
+  const promptInputs = await buildPromptInputs(testCase);
+  if (promptDumpDir) {
+    await dumpPrompt(promptDumpDir, testCase, promptInputs);
+  }
+  const cacheKey = useCache ? createCacheKey(provider, target, testCase, promptInputs) : void 0;
+  let cachedResponse;
+  if (cacheKey && cache) {
+    cachedResponse = await cache.get(cacheKey);
+  }
+  const nowFn = now ?? (() => /* @__PURE__ */ new Date());
+  const attemptBudget = (maxRetries ?? 0) + 1;
+  let attempt = 0;
+  let providerResponse = cachedResponse;
+  let lastError;
+  while (!providerResponse && attempt < attemptBudget) {
+    try {
+      providerResponse = await invokeProvider(provider, {
+        testCase,
+        target,
+        promptInputs,
+        attempt,
+        agentTimeoutMs,
+        signal
+      });
+    } catch (error) {
+      lastError = error;
+      if (isTimeoutLike(error) && attempt + 1 < attemptBudget) {
+        attempt += 1;
+        continue;
+      }
+      return buildErrorResult(testCase, target.name, nowFn(), error, promptInputs);
+    }
+  }
+  if (!providerResponse) {
+    return buildErrorResult(
+      testCase,
+      target.name,
+      nowFn(),
+      lastError ?? new Error("Provider did not return a response"),
+      promptInputs
+    );
+  }
+  if (cacheKey && cache && !cachedResponse) {
+    await cache.set(cacheKey, providerResponse);
+  }
+  const graderKind = testCase.grader ?? "heuristic";
+  const activeGrader = graders[graderKind] ?? graders.heuristic;
+  if (!activeGrader) {
+    throw new Error(`No grader registered for kind '${graderKind}'`);
+  }
+  let grade;
+  try {
+    const gradeTimestamp = nowFn();
+    grade = await activeGrader.grade({
+      testCase,
+      candidate: providerResponse.text ?? "",
+      target,
+      provider,
+      attempt,
+      promptInputs,
+      now: gradeTimestamp,
+      judgeProvider
+    });
+  } catch (error) {
+    return buildErrorResult(testCase, target.name, nowFn(), error, promptInputs);
+  }
+  const completedAt = nowFn();
+  const rawRequest = {
+    request: promptInputs.request,
+    guidelines: promptInputs.guidelines,
+    guideline_paths: testCase.guideline_paths
+  };
+  return {
+    test_id: testCase.id,
+    conversation_id: testCase.conversation_id,
+    score: grade.score,
+    hits: grade.hits,
+    misses: grade.misses,
+    model_answer: providerResponse.text ?? "",
+    expected_aspect_count: grade.expectedAspectCount,
+    target: target.name,
+    timestamp: completedAt.toISOString(),
+    reasoning: grade.reasoning,
+    raw_aspects: grade.rawAspects,
+    raw_request: rawRequest,
+    grader_raw_request: grade.graderRawRequest
+  };
+}
+function filterTestCases(testCases, testId) {
+  if (!testId) {
+    return testCases;
+  }
+  return testCases.filter((testCase) => testCase.id === testId);
+}
+function buildGraderRegistry(overrides, resolveJudgeProvider) {
+  const heuristic = overrides?.heuristic ?? new HeuristicGrader();
+  const llmJudge = overrides?.llm_judge ?? new QualityGrader({
+    resolveJudgeProvider: async (context) => {
+      if (context.judgeProvider) {
+        return context.judgeProvider;
+      }
+      return resolveJudgeProvider(context.target);
+    }
+  });
+  return {
+    ...overrides,
+    heuristic,
+    llm_judge: llmJudge
+  };
+}
+async function dumpPrompt(directory, testCase, promptInputs) {
+  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
+  const filename = `${timestamp}_${sanitizeFilename(testCase.id)}.json`;
+  const filePath = import_node_path5.default.resolve(directory, filename);
+  await (0, import_promises5.mkdir)(import_node_path5.default.dirname(filePath), { recursive: true });
+  const payload = {
+    test_id: testCase.id,
+    request: promptInputs.request,
+    guidelines: promptInputs.guidelines,
+    guideline_paths: testCase.guideline_paths
+  };
+  await (0, import_promises5.writeFile)(filePath, JSON.stringify(payload, null, 2), "utf8");
+}
+function sanitizeFilename(value) {
+  if (!value) {
+    return "prompt";
+  }
+  const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
+  return sanitized.length > 0 ? sanitized : (0, import_node_crypto2.randomUUID)();
+}
+async function invokeProvider(provider, options) {
+  const { testCase, target, promptInputs, attempt, agentTimeoutMs, signal } = options;
+  const controller = new AbortController();
+  const timeout = agentTimeoutMs ? setTimeout(() => controller.abort(), agentTimeoutMs) : void 0;
+  if (signal) {
+    signal.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  try {
+    return await provider.invoke({
+      prompt: promptInputs.request,
+      guidelines: promptInputs.guidelines,
+      attachments: testCase.guideline_paths,
+      testCaseId: testCase.id,
+      attempt,
+      metadata: {
+        target: target.name,
+        grader: testCase.grader
+      },
+      signal: controller.signal
+    });
+  } finally {
+    if (timeout !== void 0) {
+      clearTimeout(timeout);
+    }
+  }
+}
+function buildErrorResult(testCase, targetName, timestamp, error, promptInputs) {
+  const message = error instanceof Error ? error.message : String(error);
+  const rawRequest = {
+    request: promptInputs.request,
+    guidelines: promptInputs.guidelines,
+    guideline_paths: testCase.guideline_paths,
+    error: message
+  };
+  return {
+    test_id: testCase.id,
+    conversation_id: testCase.conversation_id,
+    score: 0,
+    hits: [],
+    misses: [`Error: ${message}`],
+    model_answer: `Error occurred: ${message}`,
+    expected_aspect_count: 0,
+    target: targetName,
+    timestamp: timestamp.toISOString(),
+    raw_aspects: [],
+    raw_request: rawRequest
+  };
+}
+function createCacheKey(provider, target, testCase, promptInputs) {
+  const hash = (0, import_node_crypto2.createHash)("sha256");
+  hash.update(provider.id);
+  hash.update(target.name);
+  hash.update(testCase.id);
+  hash.update(promptInputs.request);
+  hash.update(promptInputs.guidelines);
+  return hash.digest("hex");
+}
+function isTimeoutLike(error) {
+  if (!error) {
+    return false;
+  }
+  if (typeof DOMException !== "undefined" && error instanceof DOMException && error.name === "AbortError") {
+    return true;
+  }
+  if (error instanceof Error) {
+    const name = error.name?.toLowerCase();
+    const message = error.message?.toLowerCase();
+    return name.includes("timeout") || message.includes("timeout");
+  }
+  const value = String(error).toLowerCase();
+  return value.includes("timeout");
+}
+
+// src/index.ts
+function createAgentKernel() {
+  return { status: "stub" };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  GRADER_KINDS,
+  HeuristicGrader,
+  QualityGrader,
+  TEST_MESSAGE_ROLES,
+  buildPromptInputs,
+  calculateHits,
+  calculateMisses,
+  createAgentKernel,
+  createProvider,
+  ensureVSCodeSubagents,
+  extractAspects,
+  extractCodeBlocks,
+  getHitCount,
+  isErrorLike,
+  isGraderKind,
+  isGuidelineFile,
+  isJsonObject,
+  isJsonValue,
+  isTestMessage,
+  isTestMessageRole,
+  listTargetNames,
+  loadTestCases,
+  readTargetDefinitions,
+  resolveAndCreateProvider,
+  resolveTargetDefinition,
+  runEvaluation,
+  runTestCase,
+  scoreCandidateResponse
+});
+//# sourceMappingURL=index.cjs.map