mcp-eval-gateway 1.0.0

package/dist/chunk-6UIMILU4.js

@@ -0,0 +1,479 @@
+// src/init-eval-project.ts
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import { join } from "path";
+var EXISTING_FILES = [
+  "config.ts",
+  "config.mts",
+  "config.mjs",
+  "config.js",
+  "tasks.yaml"
+];
+var INIT_CONFIG = `export default {
+  model: 'gateway/anthropic/claude-sonnet-4-6',
+  threshold: 0.8,
+  mcp: {
+    url: 'http://localhost/mcp',
+    headers: { Authorization: \`Bearer \${process.env.MCP_API_KEY}\` },
+  },
+};
+`;
+var INIT_TASKS = `- name: ping
+  prompt: Call the ping tool and return its text
+  expected: pong
+  required: true
+`;
+function initEvalProject(rootDir, options) {
+  const relativeDir = options?.dir ?? "eval";
+  const evalDir = join(rootDir, relativeDir);
+  for (const filename of EXISTING_FILES) {
+    if (existsSync(join(evalDir, filename))) {
+      throw new Error(`${relativeDir}/${filename} already exists`);
+    }
+  }
+  mkdirSync(evalDir, { recursive: true });
+  writeFileSync(join(evalDir, "config.ts"), INIT_CONFIG);
+  writeFileSync(join(evalDir, "tasks.yaml"), INIT_TASKS);
+}
+
+// src/mcp-tools.ts
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { dynamicTool, jsonSchema } from "ai";
+var PACKAGE_VERSION = "0.1.0";
+function contentText(result) {
+  if (!("content" in result) || !Array.isArray(result.content)) {
+    return "";
+  }
+  return result.content.filter((part) => part.type === "text").map((part) => part.text).join("\n");
+}
+async function toolsFromMcp(options) {
+  const transport = "transport" in options ? options.transport : new StreamableHTTPClientTransport(new URL(options.url), {
+    fetch: options.fetch,
+    requestInit: options.headers ? { headers: options.headers } : void 0
+  });
+  const client = new Client({ name: "mcp-eval-gateway", version: PACKAGE_VERSION });
+  await client.connect(transport);
+  try {
+    const { tools: listed } = await client.listTools();
+    const tools = {};
+    for (const listedTool of listed) {
+      tools[listedTool.name] = dynamicTool({
+        description: listedTool.description,
+        inputSchema: jsonSchema(listedTool.inputSchema),
+        execute: async (input) => {
+          const result = await client.callTool({
+            name: listedTool.name,
+            arguments: input
+          });
+          return contentText(result);
+        }
+      });
+    }
+    return {
+      tools,
+      close: () => client.close()
+    };
+  } catch (error) {
+    await client.close();
+    throw error;
+  }
+}
+
+// src/evaluation-prompt.ts
+var EVALUATION_PROMPT = `You are an AI assistant with access to tools.
+
+When given a task, you MUST:
+1. Use the available tools to complete the task
+2. Provide summary of each step in your approach, wrapped in <summary> tags
+3. Provide feedback on the tools provided, wrapped in <feedback> tags
+4. Provide your final response, wrapped in <response> tags
+
+Summary Requirements:
+- In your <summary> tags, you must explain:
+  - The steps you took to complete the task
+  - Which tools you used, in what order, and why
+  - The inputs you provided to each tool
+  - The outputs you received from each tool
+  - A summary for how you arrived at the response
+
+Feedback Requirements:
+- In your <feedback> tags, provide constructive feedback on the tools:
+  - Comment on tool names: Are they clear and descriptive?
+  - Comment on input parameters: Are they well-documented? Are required vs optional parameters clear?
+  - Comment on descriptions: Do they accurately describe what the tool does?
+  - Comment on any errors encountered during tool usage: Did the tool fail to execute? Did the tool return too many tokens?
+  - Identify specific areas for improvement and explain WHY they would help
+  - Be specific and actionable in your suggestions
+
+Response Requirements:
+- Your response should be concise and directly address what was asked
+- Always wrap your final response in <response> tags
+- If you cannot solve the task return <response>NOT_FOUND</response>
+- For numeric responses, provide just the number
+- For IDs, provide just the ID
+- For names or text, provide the exact text requested
+- Your response should go last`;
+
+// src/resolve-model.ts
+var UNSUPPORTED_MODEL = (model) => `Unsupported model "${model}". Use one of: anthropic/<id>, openai/<id>, bedrock/<id>, gateway/<id>, or pass a LanguageModel instance.`;
+function isModuleNotFound(error) {
+  if (typeof error !== "object" || error === null) {
+    return false;
+  }
+  const code = "code" in error ? error.code : void 0;
+  if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") {
+    return true;
+  }
+  if (error instanceof Error) {
+    return error.message.includes("Cannot find module") || error.message.includes("Cannot find package");
+  }
+  return false;
+}
+function missingPeerError(model, pkg) {
+  return new Error(
+    `Model "${model}" requires the optional peer dependency "${pkg}". Install it: npm install --save-dev ${pkg}`
+  );
+}
+async function loadProvider(model, pkg, load) {
+  try {
+    return await load();
+  } catch (error) {
+    if (isModuleNotFound(error)) {
+      throw missingPeerError(model, pkg);
+    }
+    throw error;
+  }
+}
+async function resolveModel(model) {
+  const slash = model.indexOf("/");
+  if (slash === -1) {
+    throw new Error(UNSUPPORTED_MODEL(model));
+  }
+  const prefix = model.slice(0, slash);
+  const id = model.slice(slash + 1);
+  const providers = {
+    gateway: async () => id,
+    anthropic: () => loadProvider(
+      model,
+      "@ai-sdk/anthropic",
+      () => import("@ai-sdk/anthropic").then((m) => m.anthropic(id))
+    ),
+    openai: () => loadProvider(
+      model,
+      "@ai-sdk/openai",
+      () => import("@ai-sdk/openai").then((m) => m.openai(id))
+    ),
+    bedrock: () => loadProvider(
+      model,
+      "@ai-sdk/amazon-bedrock",
+      () => import("@ai-sdk/amazon-bedrock").then((m) => m.bedrock(id))
+    )
+  };
+  const resolve2 = providers[prefix];
+  if (!resolve2) {
+    throw new Error(UNSUPPORTED_MODEL(model));
+  }
+  return resolve2();
+}
+
+// src/run-evals.ts
+import { generateText, isStepCount } from "ai";
+
+// src/report.ts
+function average(values) {
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function renderReport(results) {
+  const total = results.length;
+  const correct = results.filter((result) => result.passed).length;
+  const accuracyPct = (correct / total * 100).toFixed(1);
+  const avgDurationSec = (average(results.map((result) => result.durationMs)) / 1e3).toFixed(2);
+  const toolCallCounts = results.map((result) => result.numToolCalls);
+  const totalToolCalls = toolCallCounts.reduce((sum, count) => sum + count, 0);
+  const avgToolCalls = average(toolCallCounts).toFixed(2);
+  const header = [
+    "# Evaluation Report",
+    "",
+    `- **Accuracy**: ${correct}/${total} (${accuracyPct}%)`,
+    `- **Average Duration**: ${avgDurationSec}s`,
+    `- **Average Tool Calls**: ${avgToolCalls}`,
+    `- **Total Tool Calls**: ${totalToolCalls}`
+  ].join("\n");
+  const sections = results.map((result) => {
+    const actual = result.actual ?? "N/A";
+    return [
+      `### Task: ${result.name}`,
+      "",
+      `**Prompt**: ${result.prompt}`,
+      "",
+      `**Ground Truth Response**: ${result.expected}`,
+      "",
+      `**Actual Response**: ${actual}`,
+      "",
+      `**Correct**: ${result.passed ? "\u2705" : "\u274C"}`,
+      "",
+      `**Duration**: ${(result.durationMs / 1e3).toFixed(2)}s`,
+      "",
+      `**Tool Calls**:`,
+      JSON.stringify(result.toolMetrics, null, 2),
+      "",
+      `**Summary**: ${result.summary ?? "N/A"}`,
+      "",
+      `**Feedback**: ${result.feedback ?? "N/A"}`
+    ].join("\n");
+  });
+  return [header, ...sections].join("\n\n---\n\n");
+}
+
+// src/run-evals.ts
+var defaultScorer = (actual, task) => actual === task.expected ? 1 : 0;
+function extractTag(text, tag) {
+  const regex = new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, "g");
+  let last = null;
+  let match;
+  while ((match = regex.exec(text)) !== null) {
+    last = match[1] ?? null;
+  }
+  return last === null ? null : last.trim();
+}
+function wrapTools(tools, metrics) {
+  const wrapped = {};
+  for (const [name, tool] of Object.entries(tools)) {
+    const originalExecute = tool.execute;
+    wrapped[name] = {
+      ...tool,
+      execute: async (input, options) => {
+        const start = performance.now();
+        try {
+          return await originalExecute?.(input, options);
+        } catch (error) {
+          return `Error executing tool ${name}: ${error instanceof Error ? error.message : String(error)}`;
+        } finally {
+          const bucket = metrics[name] ?? { count: 0, durationsMs: [] };
+          bucket.count += 1;
+          bucket.durationsMs.push(performance.now() - start);
+          metrics[name] = bucket;
+        }
+      }
+    };
+  }
+  return wrapped;
+}
+async function runEvals(options) {
+  if (options.tasks.length === 0) {
+    throw new Error("No tasks ran");
+  }
+  const model = typeof options.model === "string" ? await resolveModel(options.model) : options.model;
+  const results = [];
+  for (const task of options.tasks) {
+    await task.setup?.();
+    const toolMetrics = {};
+    const wrappedTools = wrapTools(options.tools, toolMetrics);
+    const started = performance.now();
+    const generated = await generateText({
+      model,
+      system: options.systemPrompt ?? EVALUATION_PROMPT,
+      prompt: task.prompt,
+      tools: wrappedTools,
+      stopWhen: isStepCount(options.maxSteps ?? 20)
+    });
+    const durationMs = performance.now() - started;
+    const actual = extractTag(generated.text, "response");
+    const scorer = task.scorer ?? options.scorer ?? defaultScorer;
+    const score = scorer(actual, task);
+    const passed = score >= 1;
+    const numToolCalls = Object.values(toolMetrics).reduce(
+      (sum, metric) => sum + metric.count,
+      0
+    );
+    results.push({
+      name: task.name,
+      prompt: task.prompt,
+      expected: task.expected,
+      actual,
+      score,
+      passed,
+      required: task.required ?? false,
+      durationMs,
+      toolMetrics,
+      numToolCalls,
+      summary: extractTag(generated.text, "summary"),
+      feedback: extractTag(generated.text, "feedback")
+    });
+  }
+  const total = results.length;
+  const correct = results.filter((result) => result.passed).length;
+  return {
+    total,
+    correct,
+    accuracy: correct / total,
+    results,
+    report: renderReport(results)
+  };
+}
+
+// src/threshold.ts
+import { appendFileSync } from "fs";
+function describeFailure(task) {
+  return `${task.name} (expected "${task.expected}", got "${task.actual}")`;
+}
+function assertEvalResult(result, options) {
+  if (result.total === 0 || result.results.length === 0) {
+    throw new Error("No tasks ran");
+  }
+  const requiredFailures = result.results.filter((task) => task.required && !task.passed);
+  if (requiredFailures.length > 0) {
+    const details = requiredFailures.map((task) => `Required task failed: ${describeFailure(task)}`).join("\n");
+    throw new Error(details);
+  }
+  const threshold = options?.threshold;
+  if (typeof threshold === "number" && result.accuracy < threshold) {
+    const failed = result.results.filter((task) => !task.passed);
+    const names = failed.map(describeFailure).join("; ");
+    throw new Error(
+      `Accuracy ${result.accuracy} is below threshold ${threshold}. Failed tasks: ${names}`
+    );
+  }
+}
+function writeGitHubSummary(result) {
+  const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+  if (!summaryPath) {
+    return false;
+  }
+  appendFileSync(summaryPath, `${result.report}
+`);
+  return true;
+}
+
+// src/run-eval-project.ts
+import { createJiti } from "jiti";
+import { existsSync as existsSync2, readFileSync } from "fs";
+import { extname, isAbsolute, join as join2, resolve } from "path";
+import { pathToFileURL } from "url";
+import { parse } from "yaml";
+var CONFIG_FILES = [
+  "config.ts",
+  "config.mts",
+  "config.mjs",
+  "config.js"
+];
+function loadEnv(rootDir, options) {
+  const envFile = options?.envFile;
+  if (envFile !== void 0) {
+    const abs = isAbsolute(envFile) ? envFile : resolve(rootDir, envFile);
+    if (!existsSync2(abs)) {
+      throw new Error(`Missing env file: ${envFile}`);
+    }
+    process.loadEnvFile(abs);
+    return;
+  }
+  const dotenv = join2(rootDir, ".env");
+  if (existsSync2(dotenv)) {
+    process.loadEnvFile(dotenv);
+  }
+}
+function resolveConfigPath(evalDir, relativeDir) {
+  for (const filename of CONFIG_FILES) {
+    const abs = join2(evalDir, filename);
+    if (existsSync2(abs)) {
+      return abs;
+    }
+  }
+  const relatives = CONFIG_FILES.map((filename) => `${relativeDir}/${filename}`);
+  throw new Error(
+    `No ${relativeDir}/config found. Expected one of: ${relatives.join(", ")}`
+  );
+}
+async function loadConfig(abs) {
+  const ext = extname(abs);
+  if (ext === ".ts" || ext === ".mts") {
+    const jiti = createJiti(abs, { moduleCache: false });
+    const loaded2 = await jiti.import(abs);
+    return loaded2.default;
+  }
+  const loaded = await import(pathToFileURL(abs).href);
+  return loaded.default;
+}
+function requireStringField(item, field, relativeDir) {
+  const value = item[field];
+  if (typeof value !== "string") {
+    throw new Error(`${relativeDir}/tasks.yaml item is missing ${field}`);
+  }
+  return value;
+}
+function loadTasks(evalDir, relativeDir) {
+  const tasksPath = join2(evalDir, "tasks.yaml");
+  if (!existsSync2(tasksPath)) {
+    throw new Error(`Missing ${relativeDir}/tasks.yaml`);
+  }
+  const parsed = parse(readFileSync(tasksPath, "utf8"));
+  if (!Array.isArray(parsed)) {
+    throw new Error(`${relativeDir}/tasks.yaml must be an array`);
+  }
+  return parsed.map((item) => {
+    if (typeof item !== "object" || item === null) {
+      throw new Error(`${relativeDir}/tasks.yaml item is missing name`);
+    }
+    const record = item;
+    return {
+      name: requireStringField(record, "name", relativeDir),
+      prompt: requireStringField(record, "prompt", relativeDir),
+      expected: requireStringField(record, "expected", relativeDir),
+      required: record.required === true
+    };
+  });
+}
+async function runEvalProject(rootDir, options) {
+  loadEnv(rootDir, options);
+  const relativeDir = options?.dir ?? "eval";
+  const evalDir = join2(rootDir, relativeDir);
+  const config = await loadConfig(resolveConfigPath(evalDir, relativeDir));
+  const tasks = loadTasks(evalDir, relativeDir);
+  const models = options?.model !== void 0 ? [options.model] : Array.isArray(config.model) ? config.model : [config.model];
+  if (models.length === 0) {
+    throw new Error("model is empty");
+  }
+  const session = await toolsFromMcp(config.mcp);
+  try {
+    const results = [];
+    for (const entry of models) {
+      results.push(
+        await runEvals({
+          model: entry,
+          tools: session.tools,
+          tasks
+        })
+      );
+    }
+    for (const result of results) {
+      writeGitHubSummary(result);
+    }
+    const assertOptions = config.threshold === void 0 ? void 0 : { threshold: config.threshold };
+    const assertErrors = [];
+    for (const result of results) {
+      try {
+        assertEvalResult(result, assertOptions);
+      } catch (err) {
+        assertErrors.push(err instanceof Error ? err : new Error(String(err)));
+      }
+    }
+    if (assertErrors.length > 0) {
+      throw new Error(assertErrors.map((err) => err.message).join("\n"));
+    }
+    return results[results.length - 1];
+  } finally {
+    await session.close();
+  }
+}
+
+export {
+  initEvalProject,
+  toolsFromMcp,
+  EVALUATION_PROMPT,
+  resolveModel,
+  runEvals,
+  assertEvalResult,
+  writeGitHubSummary,
+  runEvalProject
+};
+//# sourceMappingURL=chunk-6UIMILU4.js.map

package/dist/chunk-6UIMILU4.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/init-eval-project.ts","../src/mcp-tools.ts","../src/evaluation-prompt.ts","../src/resolve-model.ts","../src/run-evals.ts","../src/report.ts","../src/threshold.ts","../src/run-eval-project.ts"],"sourcesContent":["import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\n\nconst EXISTING_FILES = [\n  'config.ts',\n  'config.mts',\n  'config.mjs',\n  'config.js',\n  'tasks.yaml',\n] as const;\n\nconst INIT_CONFIG = `export default {\n  model: 'gateway/anthropic/claude-sonnet-4-6',\n  threshold: 0.8,\n  mcp: {\n    url: 'http://localhost/mcp',\n    headers: { Authorization: \\`Bearer \\${process.env.MCP_API_KEY}\\` },\n  },\n};\n`;\n\nconst INIT_TASKS = `- name: ping\n  prompt: Call the ping tool and return its text\n  expected: pong\n  required: true\n`;\n\nexport type InitEvalProjectOptions = {\n  dir?: string;\n};\n\nexport function initEvalProject(\n  rootDir: string,\n  options?: InitEvalProjectOptions,\n): void {\n  const relativeDir = options?.dir ?? 'eval';\n  const evalDir = join(rootDir, relativeDir);\n\n  for (const filename of EXISTING_FILES) {\n    if (existsSync(join(evalDir, filename))) {\n      throw new Error(`${relativeDir}/${filename} already exists`);\n    }\n  }\n\n  mkdirSync(evalDir, { recursive: true });\n  writeFileSync(join(evalDir, 'config.ts'), INIT_CONFIG);\n  writeFileSync(join(evalDir, 'tasks.yaml'), INIT_TASKS);\n}\n","import { Client } from '@modelcontextprotocol/sdk/client/index.js';\nimport { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';\nimport type { FetchLike, Transport } from '@modelcontextprotocol/sdk/shared/transport.js';\nimport { dynamicTool, jsonSchema, type JSONSchema7, type ToolSet } from 'ai';\n\nconst PACKAGE_VERSION = '0.1.0';\n\nexport type ToolsFromMcpOptions =\n  | {\n      url: string | URL;\n      fetch?: FetchLike;\n      headers?: Record<string, string>;\n    }\n  | {\n      transport: Transport;\n    };\n\nfunction contentText(result: Awaited<ReturnType<Client['callTool']>>): string {\n  if (!('content' in result) || !Array.isArray(result.content)) {\n    return '';\n  }\n\n  return result.content\n    .filter((part): part is { type: 'text'; text: string } => part.type === 'text')\n    .map((part) => part.text)\n    .join('\\n');\n}\n\nexport async function toolsFromMcp(\n  options: ToolsFromMcpOptions,\n): Promise<{ tools: ToolSet; close: () => Promise<void> }> {\n  const transport =\n    'transport' in options\n      ? options.transport\n      : new StreamableHTTPClientTransport(new URL(options.url), {\n          fetch: options.fetch,\n          requestInit: options.headers ? { headers: options.headers } : undefined,\n        });\n\n  const client = new Client({ name: 'mcp-eval-gateway', version: PACKAGE_VERSION });\n  await client.connect(transport);\n\n  try {\n    const { tools: listed } = await client.listTools();\n    const tools: ToolSet = {};\n\n    for (const listedTool of listed) {\n      tools[listedTool.name] = dynamicTool({\n        description: listedTool.description,\n        inputSchema: jsonSchema(listedTool.inputSchema as JSONSchema7),\n        execute: async (input) => {\n          const result = await client.callTool({\n            name: listedTool.name,\n            arguments: input as Record<string, unknown>,\n          });\n          return contentText(result);\n        },\n      });\n    }\n\n    return {\n      tools,\n      close: () => client.close(),\n    };\n  } catch (error) {\n    await client.close();\n    throw error;\n  }\n}\n","export const EVALUATION_PROMPT = `You are an AI assistant with access to tools.\n\nWhen given a task, you MUST:\n1. Use the available tools to complete the task\n2. Provide summary of each step in your approach, wrapped in <summary> tags\n3. Provide feedback on the tools provided, wrapped in <feedback> tags\n4. Provide your final response, wrapped in <response> tags\n\nSummary Requirements:\n- In your <summary> tags, you must explain:\n  - The steps you took to complete the task\n  - Which tools you used, in what order, and why\n  - The inputs you provided to each tool\n  - The outputs you received from each tool\n  - A summary for how you arrived at the response\n\nFeedback Requirements:\n- In your <feedback> tags, provide constructive feedback on the tools:\n  - Comment on tool names: Are they clear and descriptive?\n  - Comment on input parameters: Are they well-documented? Are required vs optional parameters clear?\n  - Comment on descriptions: Do they accurately describe what the tool does?\n  - Comment on any errors encountered during tool usage: Did the tool fail to execute? Did the tool return too many tokens?\n  - Identify specific areas for improvement and explain WHY they would help\n  - Be specific and actionable in your suggestions\n\nResponse Requirements:\n- Your response should be concise and directly address what was asked\n- Always wrap your final response in <response> tags\n- If you cannot solve the task return <response>NOT_FOUND</response>\n- For numeric responses, provide just the number\n- For IDs, provide just the ID\n- For names or text, provide the exact text requested\n- Your response should go last`;\n","import type { LanguageModel } from 'ai';\n\nconst UNSUPPORTED_MODEL = (model: string) =>\n  `Unsupported model \"${model}\". Use one of: anthropic/<id>, openai/<id>, bedrock/<id>, gateway/<id>, or pass a LanguageModel instance.`;\n\nfunction isModuleNotFound(error: unknown): boolean {\n  if (typeof error !== 'object' || error === null) {\n    return false;\n  }\n\n  const code = 'code' in error ? error.code : undefined;\n  if (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND') {\n    return true;\n  }\n\n  if (error instanceof Error) {\n    return (\n      error.message.includes('Cannot find module') ||\n      error.message.includes('Cannot find package')\n    );\n  }\n\n  return false;\n}\n\nfunction missingPeerError(model: string, pkg: string): Error {\n  return new Error(\n    `Model \"${model}\" requires the optional peer dependency \"${pkg}\". Install it: npm install --save-dev ${pkg}`,\n  );\n}\n\nasync function loadProvider(\n  model: string,\n  pkg: string,\n  load: () => Promise<LanguageModel>,\n): Promise<LanguageModel> {\n  try {\n    return await load();\n  } catch (error) {\n    if (isModuleNotFound(error)) {\n      throw missingPeerError(model, pkg);\n    }\n    throw error;\n  }\n}\n\nexport async function resolveModel(model: string): Promise<LanguageModel> {\n  const slash = model.indexOf('/');\n  if (slash === -1) {\n    throw new Error(UNSUPPORTED_MODEL(model));\n  }\n\n  const prefix = model.slice(0, slash);\n  const id = model.slice(slash + 1);\n\n  const providers: Record<string, () => Promise<LanguageModel>> = {\n    gateway: async () => id,\n    anthropic: () =>\n      loadProvider(model, '@ai-sdk/anthropic', () =>\n        import('@ai-sdk/anthropic').then((m) => m.anthropic(id)),\n      ),\n    openai: () =>\n      loadProvider(model, '@ai-sdk/openai', () =>\n        import('@ai-sdk/openai').then((m) => m.openai(id)),\n      ),\n    bedrock: () =>\n      loadProvider(model, '@ai-sdk/amazon-bedrock', () =>\n        import('@ai-sdk/amazon-bedrock').then((m) => m.bedrock(id)),\n      ),\n  };\n\n  const resolve = providers[prefix];\n  if (!resolve) {\n    throw new Error(UNSUPPORTED_MODEL(model));\n  }\n\n  return resolve();\n}\n","import { generateText, isStepCount, type LanguageModel, type ToolSet } from 'ai';\nimport { EVALUATION_PROMPT } from './evaluation-prompt';\nimport { renderReport } from './report';\nimport { resolveModel } from './resolve-model';\nimport type { EvalRunResult, EvalTask, TaskResult, ToolMetrics } from './types';\n\nexport type RunEvalsOptions = {\n  model: string | LanguageModel;\n  tools: ToolSet;\n  tasks: EvalTask[];\n  maxSteps?: number;\n  systemPrompt?: string;\n  scorer?: (actual: string | null, task: EvalTask) => number;\n};\n\nconst defaultScorer = (actual: string | null, task: EvalTask): number =>\n  actual === task.expected ? 1 : 0;\n\nfunction extractTag(text: string, tag: string): string | null {\n  const regex = new RegExp(`<${tag}>([\\\\s\\\\S]*?)</${tag}>`, 'g');\n  let last: string | null = null;\n  let match: RegExpExecArray | null;\n  while ((match = regex.exec(text)) !== null) {\n    last = match[1] ?? null;\n  }\n  return last === null ? null : last.trim();\n}\n\nfunction wrapTools(tools: ToolSet, metrics: ToolMetrics): ToolSet {\n  const wrapped: ToolSet = {};\n\n  for (const [name, tool] of Object.entries(tools)) {\n    const originalExecute = tool.execute;\n    wrapped[name] = {\n      ...tool,\n      execute: async (input, options) => {\n        const start = performance.now();\n        try {\n          return await originalExecute?.(input, options);\n        } catch (error) {\n          return `Error executing tool ${name}: ${\n            error instanceof Error ? error.message : String(error)\n          }`;\n        } finally {\n          const bucket = metrics[name] ?? { count: 0, durationsMs: [] };\n          bucket.count += 1;\n          bucket.durationsMs.push(performance.now() - start);\n          metrics[name] = bucket;\n        }\n      },\n    };\n  }\n\n  return wrapped;\n}\n\nexport async function runEvals(options: RunEvalsOptions): Promise<EvalRunResult> {\n  if (options.tasks.length === 0) {\n    throw new Error('No tasks ran');\n  }\n\n  const model =\n    typeof options.model === 'string' ? await resolveModel(options.model) : options.model;\n\n  const results: TaskResult[] = [];\n\n  for (const task of options.tasks) {\n    await task.setup?.();\n\n    const toolMetrics: ToolMetrics = {};\n    const wrappedTools = wrapTools(options.tools, toolMetrics);\n    const started = performance.now();\n\n    const generated = await generateText({\n      model,\n      system: options.systemPrompt ?? EVALUATION_PROMPT,\n      prompt: task.prompt,\n      tools: wrappedTools,\n      stopWhen: isStepCount(options.maxSteps ?? 20),\n    });\n\n    const durationMs = performance.now() - started;\n    const actual = extractTag(generated.text, 'response');\n    const scorer = task.scorer ?? options.scorer ?? defaultScorer;\n    const score = scorer(actual, task);\n    const passed = score >= 1;\n    const numToolCalls = Object.values(toolMetrics).reduce(\n      (sum, metric) => sum + metric.count,\n      0,\n    );\n\n    results.push({\n      name: task.name,\n      prompt: task.prompt,\n      expected: task.expected,\n      actual,\n      score,\n      passed,\n      required: task.required ?? false,\n      durationMs,\n      toolMetrics,\n      numToolCalls,\n      summary: extractTag(generated.text, 'summary'),\n      feedback: extractTag(generated.text, 'feedback'),\n    });\n  }\n\n  const total = results.length;\n  const correct = results.filter((result) => result.passed).length;\n\n  return {\n    total,\n    correct,\n    accuracy: correct / total,\n    results,\n    report: renderReport(results),\n  };\n}\n","import type { TaskResult } from './types';\n\nfunction average(values: number[]): number {\n  return values.reduce((sum, value) => sum + value, 0) / values.length;\n}\n\nexport function renderReport(results: TaskResult[]): string {\n  const total = results.length;\n  const correct = results.filter((result) => result.passed).length;\n  const accuracyPct = ((correct / total) * 100).toFixed(1);\n  const avgDurationSec = (average(results.map((result) => result.durationMs)) / 1000).toFixed(2);\n  const toolCallCounts = results.map((result) => result.numToolCalls);\n  const totalToolCalls = toolCallCounts.reduce((sum, count) => sum + count, 0);\n  const avgToolCalls = average(toolCallCounts).toFixed(2);\n\n  const header = [\n    '# Evaluation Report',\n    '',\n    `- **Accuracy**: ${correct}/${total} (${accuracyPct}%)`,\n    `- **Average Duration**: ${avgDurationSec}s`,\n    `- **Average Tool Calls**: ${avgToolCalls}`,\n    `- **Total Tool Calls**: ${totalToolCalls}`,\n  ].join('\\n');\n\n  const sections = results.map((result) => {\n    const actual = result.actual ?? 'N/A';\n    return [\n      `### Task: ${result.name}`,\n      '',\n      `**Prompt**: ${result.prompt}`,\n      '',\n      `**Ground Truth Response**: ${result.expected}`,\n      '',\n      `**Actual Response**: ${actual}`,\n      '',\n      `**Correct**: ${result.passed ? '✅' : '❌'}`,\n      '',\n      `**Duration**: ${(result.durationMs / 1000).toFixed(2)}s`,\n      '',\n      `**Tool Calls**:`,\n      JSON.stringify(result.toolMetrics, null, 2),\n      '',\n      `**Summary**: ${result.summary ?? 'N/A'}`,\n      '',\n      `**Feedback**: ${result.feedback ?? 'N/A'}`,\n    ].join('\\n');\n  });\n\n  return [header, ...sections].join('\\n\\n---\\n\\n');\n}\n","import { appendFileSync } from 'node:fs';\nimport type { EvalRunResult, TaskResult } from './types';\n\nfunction describeFailure(task: TaskResult): string {\n  return `${task.name} (expected \"${task.expected}\", got \"${task.actual}\")`;\n}\n\nexport function assertEvalResult(\n  result: EvalRunResult,\n  options?: { threshold?: number },\n): void {\n  if (result.total === 0 || result.results.length === 0) {\n    throw new Error('No tasks ran');\n  }\n\n  const requiredFailures = result.results.filter((task) => task.required && !task.passed);\n  if (requiredFailures.length > 0) {\n    const details = requiredFailures\n      .map((task) => `Required task failed: ${describeFailure(task)}`)\n      .join('\\n');\n    throw new Error(details);\n  }\n\n  const threshold = options?.threshold;\n  if (typeof threshold === 'number' && result.accuracy < threshold) {\n    const failed = result.results.filter((task) => !task.passed);\n    const names = failed.map(describeFailure).join('; ');\n    throw new Error(\n      `Accuracy ${result.accuracy} is below threshold ${threshold}. Failed tasks: ${names}`,\n    );\n  }\n}\n\nexport function writeGitHubSummary(result: EvalRunResult): boolean {\n  const summaryPath = process.env.GITHUB_STEP_SUMMARY;\n  if (!summaryPath) {\n    return false;\n  }\n\n  appendFileSync(summaryPath, `${result.report}\\n`);\n  return true;\n}\n","import type { LanguageModel } from 'ai';\nimport { createJiti } from 'jiti';\nimport { existsSync, readFileSync } from 'node:fs';\nimport { extname, isAbsolute, join, resolve } from 'node:path';\nimport { pathToFileURL } from 'node:url';\nimport { parse } from 'yaml';\nimport { toolsFromMcp, type ToolsFromMcpOptions } from './mcp-tools';\nimport { runEvals } from './run-evals';\nimport { assertEvalResult, writeGitHubSummary } from './threshold';\nimport type { EvalRunResult, EvalTask } from './types';\n\nconst CONFIG_FILES = [\n  'config.ts',\n  'config.mts',\n  'config.mjs',\n  'config.js',\n] as const;\n\nexport type RunEvalProjectOptions = {\n  dir?: string;\n  envFile?: string;\n  model?: string;\n};\n\ntype EvalProjectModel = string | LanguageModel;\n\ntype EvalProjectConfig = {\n  model: EvalProjectModel | EvalProjectModel[];\n  threshold?: number;\n  mcp: ToolsFromMcpOptions;\n};\n\nfunction loadEnv(rootDir: string, options?: RunEvalProjectOptions): void {\n  const envFile = options?.envFile;\n  if (envFile !== undefined) {\n    const abs = isAbsolute(envFile) ? envFile : resolve(rootDir, envFile);\n    if (!existsSync(abs)) {\n      throw new Error(`Missing env file: ${envFile}`);\n    }\n    process.loadEnvFile(abs);\n    return;\n  }\n\n  const dotenv = join(rootDir, '.env');\n  if (existsSync(dotenv)) {\n    process.loadEnvFile(dotenv);\n  }\n}\n\nfunction resolveConfigPath(evalDir: string, relativeDir: string): string {\n  for (const filename of CONFIG_FILES) {\n    const abs = join(evalDir, filename);\n    if (existsSync(abs)) {\n      return abs;\n    }\n  }\n  const relatives = CONFIG_FILES.map((filename) => `${relativeDir}/${filename}`);\n  throw new Error(\n    `No ${relativeDir}/config found. Expected one of: ${relatives.join(', ')}`,\n  );\n}\n\nasync function loadConfig(abs: string): Promise<EvalProjectConfig> {\n  const ext = extname(abs);\n  if (ext === '.ts' || ext === '.mts') {\n    const jiti = createJiti(abs, { moduleCache: false });\n    const loaded = (await jiti.import(abs)) as { default: EvalProjectConfig };\n    return loaded.default;\n  }\n  const loaded = (await import(pathToFileURL(abs).href)) as {\n    default: EvalProjectConfig;\n  };\n  return loaded.default;\n}\n\nfunction requireStringField(\n  item: Record<string, unknown>,\n  field: string,\n  relativeDir: string,\n): string {\n  const value = item[field];\n  if (typeof value !== 'string') {\n    throw new Error(`${relativeDir}/tasks.yaml item is missing ${field}`);\n  }\n  return value;\n}\n\nfunction loadTasks(evalDir: string, relativeDir: string): EvalTask[] {\n  const tasksPath = join(evalDir, 'tasks.yaml');\n  if (!existsSync(tasksPath)) {\n    throw new Error(`Missing ${relativeDir}/tasks.yaml`);\n  }\n\n  const parsed: unknown = parse(readFileSync(tasksPath, 'utf8'));\n  if (!Array.isArray(parsed)) {\n    throw new Error(`${relativeDir}/tasks.yaml must be an array`);\n  }\n\n  return parsed.map((item: unknown) => {\n    if (typeof item !== 'object' || item === null) {\n      throw new Error(`${relativeDir}/tasks.yaml item is missing name`);\n    }\n    const record = item as Record<string, unknown>;\n    return {\n      name: requireStringField(record, 'name', relativeDir),\n      prompt: requireStringField(record, 'prompt', relativeDir),\n      expected: requireStringField(record, 'expected', relativeDir),\n      required: record.required === true,\n    };\n  });\n}\n\nexport async function runEvalProject(\n  rootDir: string,\n  options?: RunEvalProjectOptions,\n): Promise<EvalRunResult> {\n  loadEnv(rootDir, options);\n\n  const relativeDir = options?.dir ?? 'eval';\n  const evalDir = join(rootDir, relativeDir);\n  const config = await loadConfig(resolveConfigPath(evalDir, relativeDir));\n  const tasks = loadTasks(evalDir, relativeDir);\n  const models = options?.model !== undefined\n    ? [options.model]\n    : Array.isArray(config.model)\n      ? config.model\n      : [config.model];\n  if (models.length === 0) {\n    throw new Error('model is empty');\n  }\n\n  const session = await toolsFromMcp(config.mcp);\n  try {\n    const results: EvalRunResult[] = [];\n    for (const entry of models) {\n      results.push(\n        await runEvals({\n          model: entry,\n          tools: session.tools,\n          tasks,\n        }),\n      );\n    }\n\n    for (const result of results) {\n      writeGitHubSummary(result);\n    }\n\n    const assertOptions =\n      config.threshold === undefined ? undefined : { threshold: config.threshold };\n    const assertErrors: Error[] = [];\n    for (const result of results) {\n      try {\n        assertEvalResult(result, assertOptions);\n      } catch (err) {\n        assertErrors.push(err instanceof Error ? err : new Error(String(err)));\n      }\n    }\n    if (assertErrors.length > 0) {\n      throw new Error(assertErrors.map((err) => err.message).join('\\n'));\n    }\n\n    return results[results.length - 1];\n  } finally {\n    await session.close();\n  }\n}\n"],"mappings":";AAAA,SAAS,YAAY,WAAW,qBAAqB;AACrD,SAAS,YAAY;AAErB,IAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,cAAc;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAUpB,IAAM,aAAa;AAAA;AAAA;AAAA;AAAA;AAUZ,SAAS,gBACd,SACA,SACM;AACN,QAAM,cAAc,SAAS,OAAO;AACpC,QAAM,UAAU,KAAK,SAAS,WAAW;AAEzC,aAAW,YAAY,gBAAgB;AACrC,QAAI,WAAW,KAAK,SAAS,QAAQ,CAAC,GAAG;AACvC,YAAM,IAAI,MAAM,GAAG,WAAW,IAAI,QAAQ,iBAAiB;AAAA,IAC7D;AAAA,EACF;AAEA,YAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,gBAAc,KAAK,SAAS,WAAW,GAAG,WAAW;AACrD,gBAAc,KAAK,SAAS,YAAY,GAAG,UAAU;AACvD;;;AC/CA,SAAS,cAAc;AACvB,SAAS,qCAAqC;AAE9C,SAAS,aAAa,kBAAkD;AAExE,IAAM,kBAAkB;AAYxB,SAAS,YAAY,QAAyD;AAC5E,MAAI,EAAE,aAAa,WAAW,CAAC,MAAM,QAAQ,OAAO,OAAO,GAAG;AAC5D,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,QACX,OAAO,CAAC,SAAiD,KAAK,SAAS,MAAM,EAC7E,IAAI,CAAC,SAAS,KAAK,IAAI,EACvB,KAAK,IAAI;AACd;AAEA,eAAsB,aACpB,SACyD;AACzD,QAAM,YACJ,eAAe,UACX,QAAQ,YACR,IAAI,8BAA8B,IAAI,IAAI,QAAQ,GAAG,GAAG;AAAA,IACtD,OAAO,QAAQ;AAAA,IACf,aAAa,QAAQ,UAAU,EAAE,SAAS,QAAQ,QAAQ,IAAI;AAAA,EAChE,CAAC;AAEP,QAAM,SAAS,IAAI,OAAO,EAAE,MAAM,oBAAoB,SAAS,gBAAgB,CAAC;AAChF,QAAM,OAAO,QAAQ,SAAS;AAE9B,MAAI;AACF,UAAM,EAAE,OAAO,OAAO,IAAI,MAAM,OAAO,UAAU;AACjD,UAAM,QAAiB,CAAC;AAExB,eAAW,cAAc,QAAQ;AAC/B,YAAM,WAAW,IAAI,IAAI,YAAY;AAAA,QACnC,aAAa,WAAW;AAAA,QACxB,aAAa,WAAW,WAAW,WAA0B;AAAA,QAC7D,SAAS,OAAO,UAAU;AACxB,gBAAM,SAAS,MAAM,OAAO,SAAS;AAAA,YACnC,MAAM,WAAW;AAAA,YACjB,WAAW;AAAA,UACb,CAAC;AACD,iBAAO,YAAY,MAAM;AAAA,QAC3B;AAAA,MACF,CAAC;AAAA,IACH;AAEA,WAAO;AAAA,MACL;AAAA,MACA,OAAO,MAAM,OAAO,MAAM;AAAA,IAC5B;AAAA,EACF,SAAS,OAAO;AACd,UAAM,OAAO,MAAM;AACnB,UAAM;AAAA,EACR;AACF;;;ACpEO,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACEjC,IAAM,oBAAoB,CAAC,UACzB,sBAAsB,KAAK;AAE7B,SAAS,iBAAiB,OAAyB;AACjD,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AAEA,QAAM,OAAO,UAAU,QAAQ,MAAM,OAAO;AAC5C,MAAI,SAAS,0BAA0B,SAAS,oBAAoB;AAClE,WAAO;AAAA,EACT;AAEA,MAAI,iBAAiB,OAAO;AAC1B,WACE,MAAM,QAAQ,SAAS,oBAAoB,KAC3C,MAAM,QAAQ,SAAS,qBAAqB;AAAA,EAEhD;AAEA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAAe,KAAoB;AAC3D,SAAO,IAAI;AAAA,IACT,UAAU,KAAK,4CAA4C,GAAG,yCAAyC,GAAG;AAAA,EAC5G;AACF;AAEA,eAAe,aACb,OACA,KACA,MACwB;AACxB,MAAI;AACF,WAAO,MAAM,KAAK;AAAA,EACpB,SAAS,OAAO;AACd,QAAI,iBAAiB,KAAK,GAAG;AAC3B,YAAM,iBAAiB,OAAO,GAAG;AAAA,IACnC;AACA,UAAM;AAAA,EACR;AACF;AAEA,eAAsB,aAAa,OAAuC;AACxE,QAAM,QAAQ,MAAM,QAAQ,GAAG;AAC/B,MAAI,UAAU,IAAI;AAChB,UAAM,IAAI,MAAM,kBAAkB,KAAK,CAAC;AAAA,EAC1C;AAEA,QAAM,SAAS,MAAM,MAAM,GAAG,KAAK;AACnC,QAAM,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEhC,QAAM,YAA0D;AAAA,IAC9D,SAAS,YAAY;AAAA,IACrB,WAAW,MACT;AAAA,MAAa;AAAA,MAAO;AAAA,MAAqB,MACvC,OAAO,mBAAmB,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC;AAAA,IACzD;AAAA,IACF,QAAQ,MACN;AAAA,MAAa;AAAA,MAAO;AAAA,MAAkB,MACpC,OAAO,gBAAgB,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;AAAA,IACnD;AAAA,IACF,SAAS,MACP;AAAA,MAAa;AAAA,MAAO;AAAA,MAA0B,MAC5C,OAAO,wBAAwB,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;AAAA,IAC5D;AAAA,EACJ;AAEA,QAAMA,WAAU,UAAU,MAAM;AAChC,MAAI,CAACA,UAAS;AACZ,UAAM,IAAI,MAAM,kBAAkB,KAAK,CAAC;AAAA,EAC1C;AAEA,SAAOA,SAAQ;AACjB;;;AC7EA,SAAS,cAAc,mBAAqD;;;ACE5E,SAAS,QAAQ,QAA0B;AACzC,SAAO,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAChE;AAEO,SAAS,aAAa,SAA+B;AAC1D,QAAM,QAAQ,QAAQ;AACtB,QAAM,UAAU,QAAQ,OAAO,CAAC,WAAW,OAAO,MAAM,EAAE;AAC1D,QAAM,eAAgB,UAAU,QAAS,KAAK,QAAQ,CAAC;AACvD,QAAM,kBAAkB,QAAQ,QAAQ,IAAI,CAAC,WAAW,OAAO,UAAU,CAAC,IAAI,KAAM,QAAQ,CAAC;AAC7F,QAAM,iBAAiB,QAAQ,IAAI,CAAC,WAAW,OAAO,YAAY;AAClE,QAAM,iBAAiB,eAAe,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC;AAC3E,QAAM,eAAe,QAAQ,cAAc,EAAE,QAAQ,CAAC;AAEtD,QAAM,SAAS;AAAA,IACb;AAAA,IACA;AAAA,IACA,mBAAmB,OAAO,IAAI,KAAK,KAAK,WAAW;AAAA,IACnD,2BAA2B,cAAc;AAAA,IACzC,6BAA6B,YAAY;AAAA,IACzC,2BAA2B,cAAc;AAAA,EAC3C,EAAE,KAAK,IAAI;AAEX,QAAM,WAAW,QAAQ,IAAI,CAAC,WAAW;AACvC,UAAM,SAAS,OAAO,UAAU;AAChC,WAAO;AAAA,MACL,aAAa,OAAO,IAAI;AAAA,MACxB;AAAA,MACA,eAAe,OAAO,MAAM;AAAA,MAC5B;AAAA,MACA,8BAA8B,OAAO,QAAQ;AAAA,MAC7C;AAAA,MACA,wBAAwB,MAAM;AAAA,MAC9B;AAAA,MACA,gBAAgB,OAAO,SAAS,WAAM,QAAG;AAAA,MACzC;AAAA,MACA,kBAAkB,OAAO,aAAa,KAAM,QAAQ,CAAC,CAAC;AAAA,MACtD;AAAA,MACA;AAAA,MACA,KAAK,UAAU,OAAO,aAAa,MAAM,CAAC;AAAA,MAC1C;AAAA,MACA,gBAAgB,OAAO,WAAW,KAAK;AAAA,MACvC;AAAA,MACA,iBAAiB,OAAO,YAAY,KAAK;AAAA,IAC3C,EAAE,KAAK,IAAI;AAAA,EACb,CAAC;AAED,SAAO,CAAC,QAAQ,GAAG,QAAQ,EAAE,KAAK,aAAa;AACjD;;;ADlCA,IAAM,gBAAgB,CAAC,QAAuB,SAC5C,WAAW,KAAK,WAAW,IAAI;AAEjC,SAAS,WAAW,MAAc,KAA4B;AAC5D,QAAM,QAAQ,IAAI,OAAO,IAAI,GAAG,kBAAkB,GAAG,KAAK,GAAG;AAC7D,MAAI,OAAsB;AAC1B,MAAI;AACJ,UAAQ,QAAQ,MAAM,KAAK,IAAI,OAAO,MAAM;AAC1C,WAAO,MAAM,CAAC,KAAK;AAAA,EACrB;AACA,SAAO,SAAS,OAAO,OAAO,KAAK,KAAK;AAC1C;AAEA,SAAS,UAAU,OAAgB,SAA+B;AAChE,QAAM,UAAmB,CAAC;AAE1B,aAAW,CAAC,MAAM,IAAI,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAM,kBAAkB,KAAK;AAC7B,YAAQ,IAAI,IAAI;AAAA,MACd,GAAG;AAAA,MACH,SAAS,OAAO,OAAO,YAAY;AACjC,cAAM,QAAQ,YAAY,IAAI;AAC9B,YAAI;AACF,iBAAO,MAAM,kBAAkB,OAAO,OAAO;AAAA,QAC/C,SAAS,OAAO;AACd,iBAAO,wBAAwB,IAAI,KACjC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CACvD;AAAA,QACF,UAAE;AACA,gBAAM,SAAS,QAAQ,IAAI,KAAK,EAAE,OAAO,GAAG,aAAa,CAAC,EAAE;AAC5D,iBAAO,SAAS;AAChB,iBAAO,YAAY,KAAK,YAAY,IAAI,IAAI,KAAK;AACjD,kBAAQ,IAAI,IAAI;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAsB,SAAS,SAAkD;AAC/E,MAAI,QAAQ,MAAM,WAAW,GAAG;AAC9B,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,QACJ,OAAO,QAAQ,UAAU,WAAW,MAAM,aAAa,QAAQ,KAAK,IAAI,QAAQ;AAElF,QAAM,UAAwB,CAAC;AAE/B,aAAW,QAAQ,QAAQ,OAAO;AAChC,UAAM,KAAK,QAAQ;AAEnB,UAAM,cAA2B,CAAC;AAClC,UAAM,eAAe,UAAU,QAAQ,OAAO,WAAW;AACzD,UAAM,UAAU,YAAY,IAAI;AAEhC,UAAM,YAAY,MAAM,aAAa;AAAA,MACnC;AAAA,MACA,QAAQ,QAAQ,gBAAgB;AAAA,MAChC,QAAQ,KAAK;AAAA,MACb,OAAO;AAAA,MACP,UAAU,YAAY,QAAQ,YAAY,EAAE;AAAA,IAC9C,CAAC;AAED,UAAM,aAAa,YAAY,IAAI,IAAI;AACvC,UAAM,SAAS,WAAW,UAAU,MAAM,UAAU;AACpD,UAAM,SAAS,KAAK,UAAU,QAAQ,UAAU;AAChD,UAAM,QAAQ,OAAO,QAAQ,IAAI;AACjC,UAAM,SAAS,SAAS;AACxB,UAAM,eAAe,OAAO,OAAO,WAAW,EAAE;AAAA,MAC9C,CAAC,KAAK,WAAW,MAAM,OAAO;AAAA,MAC9B;AAAA,IACF;AAEA,YAAQ,KAAK;AAAA,MACX,MAAM,KAAK;AAAA,MACX,QAAQ,KAAK;AAAA,MACb,UAAU,KAAK;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,KAAK,YAAY;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,WAAW,UAAU,MAAM,SAAS;AAAA,MAC7C,UAAU,WAAW,UAAU,MAAM,UAAU;AAAA,IACjD,CAAC;AAAA,EACH;AAEA,QAAM,QAAQ,QAAQ;AACtB,QAAM,UAAU,QAAQ,OAAO,CAAC,WAAW,OAAO,MAAM,EAAE;AAE1D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,QAAQ,aAAa,OAAO;AAAA,EAC9B;AACF;;;AErHA,SAAS,sBAAsB;AAG/B,SAAS,gBAAgB,MAA0B;AACjD,SAAO,GAAG,KAAK,IAAI,eAAe,KAAK,QAAQ,WAAW,KAAK,MAAM;AACvE;AAEO,SAAS,iBACd,QACA,SACM;AACN,MAAI,OAAO,UAAU,KAAK,OAAO,QAAQ,WAAW,GAAG;AACrD,UAAM,IAAI,MAAM,cAAc;AAAA,EAChC;AAEA,QAAM,mBAAmB,OAAO,QAAQ,OAAO,CAAC,SAAS,KAAK,YAAY,CAAC,KAAK,MAAM;AACtF,MAAI,iBAAiB,SAAS,GAAG;AAC/B,UAAM,UAAU,iBACb,IAAI,CAAC,SAAS,yBAAyB,gBAAgB,IAAI,CAAC,EAAE,EAC9D,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,OAAO;AAAA,EACzB;AAEA,QAAM,YAAY,SAAS;AAC3B,MAAI,OAAO,cAAc,YAAY,OAAO,WAAW,WAAW;AAChE,UAAM,SAAS,OAAO,QAAQ,OAAO,CAAC,SAAS,CAAC,KAAK,MAAM;AAC3D,UAAM,QAAQ,OAAO,IAAI,eAAe,EAAE,KAAK,IAAI;AACnD,UAAM,IAAI;AAAA,MACR,YAAY,OAAO,QAAQ,uBAAuB,SAAS,mBAAmB,KAAK;AAAA,IACrF;AAAA,EACF;AACF;AAEO,SAAS,mBAAmB,QAAgC;AACjE,QAAM,cAAc,QAAQ,IAAI;AAChC,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AAEA,iBAAe,aAAa,GAAG,OAAO,MAAM;AAAA,CAAI;AAChD,SAAO;AACT;;;ACxCA,SAAS,kBAAkB;AAC3B,SAAS,cAAAC,aAAY,oBAAoB;AACzC,SAAS,SAAS,YAAY,QAAAC,OAAM,eAAe;AACnD,SAAS,qBAAqB;AAC9B,SAAS,aAAa;AAMtB,IAAM,eAAe;AAAA,EACnB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAgBA,SAAS,QAAQ,SAAiB,SAAuC;AACvE,QAAM,UAAU,SAAS;AACzB,MAAI,YAAY,QAAW;AACzB,UAAM,MAAM,WAAW,OAAO,IAAI,UAAU,QAAQ,SAAS,OAAO;AACpE,QAAI,CAACC,YAAW,GAAG,GAAG;AACpB,YAAM,IAAI,MAAM,qBAAqB,OAAO,EAAE;AAAA,IAChD;AACA,YAAQ,YAAY,GAAG;AACvB;AAAA,EACF;AAEA,QAAM,SAASC,MAAK,SAAS,MAAM;AACnC,MAAID,YAAW,MAAM,GAAG;AACtB,YAAQ,YAAY,MAAM;AAAA,EAC5B;AACF;AAEA,SAAS,kBAAkB,SAAiB,aAA6B;AACvE,aAAW,YAAY,cAAc;AACnC,UAAM,MAAMC,MAAK,SAAS,QAAQ;AAClC,QAAID,YAAW,GAAG,GAAG;AACnB,aAAO;AAAA,IACT;AAAA,EACF;AACA,QAAM,YAAY,aAAa,IAAI,CAAC,aAAa,GAAG,WAAW,IAAI,QAAQ,EAAE;AAC7E,QAAM,IAAI;AAAA,IACR,MAAM,WAAW,mCAAmC,UAAU,KAAK,IAAI,CAAC;AAAA,EAC1E;AACF;AAEA,eAAe,WAAW,KAAyC;AACjE,QAAM,MAAM,QAAQ,GAAG;AACvB,MAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,UAAM,OAAO,WAAW,KAAK,EAAE,aAAa,MAAM,CAAC;AACnD,UAAME,UAAU,MAAM,KAAK,OAAO,GAAG;AACrC,WAAOA,QAAO;AAAA,EAChB;AACA,QAAM,SAAU,MAAM,OAAO,cAAc,GAAG,EAAE;AAGhD,SAAO,OAAO;AAChB;AAEA,SAAS,mBACP,MACA,OACA,aACQ;AACR,QAAM,QAAQ,KAAK,KAAK;AACxB,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,IAAI,MAAM,GAAG,WAAW,+BAA+B,KAAK,EAAE;AAAA,EACtE;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAAiB,aAAiC;AACnE,QAAM,YAAYD,MAAK,SAAS,YAAY;AAC5C,MAAI,CAACD,YAAW,SAAS,GAAG;AAC1B,UAAM,IAAI,MAAM,WAAW,WAAW,aAAa;AAAA,EACrD;AAEA,QAAM,SAAkB,MAAM,aAAa,WAAW,MAAM,CAAC;AAC7D,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAM,IAAI,MAAM,GAAG,WAAW,8BAA8B;AAAA,EAC9D;AAEA,SAAO,OAAO,IAAI,CAAC,SAAkB;AACnC,QAAI,OAAO,SAAS,YAAY,SAAS,MAAM;AAC7C,YAAM,IAAI,MAAM,GAAG,WAAW,kCAAkC;AAAA,IAClE;AACA,UAAM,SAAS;AACf,WAAO;AAAA,MACL,MAAM,mBAAmB,QAAQ,QAAQ,WAAW;AAAA,MACpD,QAAQ,mBAAmB,QAAQ,UAAU,WAAW;AAAA,MACxD,UAAU,mBAAmB,QAAQ,YAAY,WAAW;AAAA,MAC5D,UAAU,OAAO,aAAa;AAAA,IAChC;AAAA,EACF,CAAC;AACH;AAEA,eAAsB,eACpB,SACA,SACwB;AACxB,UAAQ,SAAS,OAAO;AAExB,QAAM,cAAc,SAAS,OAAO;AACpC,QAAM,UAAUC,MAAK,SAAS,WAAW;AACzC,QAAM,SAAS,MAAM,WAAW,kBAAkB,SAAS,WAAW,CAAC;AACvE,QAAM,QAAQ,UAAU,SAAS,WAAW;AAC5C,QAAM,SAAS,SAAS,UAAU,SAC9B,CAAC,QAAQ,KAAK,IACd,MAAM,QAAQ,OAAO,KAAK,IACxB,OAAO,QACP,CAAC,OAAO,KAAK;AACnB,MAAI,OAAO,WAAW,GAAG;AACvB,UAAM,IAAI,MAAM,gBAAgB;AAAA,EAClC;AAEA,QAAM,UAAU,MAAM,aAAa,OAAO,GAAG;AAC7C,MAAI;AACF,UAAM,UAA2B,CAAC;AAClC,eAAW,SAAS,QAAQ;AAC1B,cAAQ;AAAA,QACN,MAAM,SAAS;AAAA,UACb,OAAO;AAAA,UACP,OAAO,QAAQ;AAAA,UACf;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,eAAW,UAAU,SAAS;AAC5B,yBAAmB,MAAM;AAAA,IAC3B;AAEA,UAAM,gBACJ,OAAO,cAAc,SAAY,SAAY,EAAE,WAAW,OAAO,UAAU;AAC7E,UAAM,eAAwB,CAAC;AAC/B,eAAW,UAAU,SAAS;AAC5B,UAAI;AACF,yBAAiB,QAAQ,aAAa;AAAA,MACxC,SAAS,KAAK;AACZ,qBAAa,KAAK,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,CAAC;AAAA,MACvE;AAAA,IACF;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,IAAI,MAAM,aAAa,IAAI,CAAC,QAAQ,IAAI,OAAO,EAAE,KAAK,IAAI,CAAC;AAAA,IACnE;AAEA,WAAO,QAAQ,QAAQ,SAAS,CAAC;AAAA,EACnC,UAAE;AACA,UAAM,QAAQ,MAAM;AAAA,EACtB;AACF;","names":["resolve","existsSync","join","existsSync","join","loaded"]}