@kat-ai/eval 0.1.0

package/dist/index.js

@@ -0,0 +1,1032 @@
+import { Pinecone } from '@pinecone-database/pinecone';
+import { createOpenAI } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
+import { z } from 'zod';
+import { resolveDefaultOpenAiChatModelId } from '@kat/core';
+
+// src/introspection/index.ts
+var GradeSchema = z.object({
+  score: z.number().min(0).max(100).describe("Score from 0-100"),
+  reasoning: z.string().describe("Explanation for the score"),
+  examples: z.array(z.string()).optional().describe("Specific examples that influenced the score")
+});
+var MultiCriteriaGradeSchema = z.object({
+  scores: z.array(z.object({
+    criterion: z.string(),
+    score: z.number().min(0).max(100),
+    reasoning: z.string()
+  })),
+  overallReasoning: z.string()
+});
+function createLLMGrader(config = {}) {
+  const apiKey = config.openaiApiKey || process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error("OPENAI_API_KEY is required for LLM grading");
+  }
+  const openai = createOpenAI({ apiKey });
+  const model = config.model || resolveDefaultOpenAiChatModelId();
+  const temperature = config.temperature ?? 0.1;
+  return {
+    async grade(content, criterion, rubric) {
+      const prompt = `You are an expert evaluator. Grade the following content on the criterion "${criterion}".
+
+${rubric ? `Rubric: ${rubric}
+` : ""}
+Content to evaluate:
+"""
+${content}
+"""
+
+Provide a score from 0-100 and explain your reasoning.`;
+      const result = await generateObject({
+        model: openai(model),
+        schema: GradeSchema,
+        prompt,
+        temperature
+      });
+      return result.object;
+    },
+    async gradeMultiple(content, criteria) {
+      const criteriaDescription = criteria.map((c) => `- ${c.name}: ${c.description} (weight: ${c.weight})`).join("\n");
+      const prompt = `You are an expert evaluator. Grade the following content on multiple criteria.
+
+Criteria:
+${criteriaDescription}
+
+Content to evaluate:
+"""
+${content}
+"""
+
+For each criterion, provide a score from 0-100 and explain your reasoning.`;
+      const result = await generateObject({
+        model: openai(model),
+        schema: MultiCriteriaGradeSchema,
+        prompt,
+        temperature
+      });
+      return result.object.scores.map((s) => ({
+        criterion: s.criterion,
+        score: s.score,
+        reasoning: s.reasoning
+      }));
+    },
+    async gradeRelevance(query, content) {
+      const prompt = `You are an expert evaluator. Grade how relevant the following content is to the given query.
+
+Query: "${query}"
+
+Content:
+"""
+${content}
+"""
+
+A score of 100 means the content directly and completely answers the query.
+A score of 0 means the content is completely irrelevant.
+
+Provide a score from 0-100 and explain your reasoning.`;
+      const result = await generateObject({
+        model: openai(model),
+        schema: GradeSchema,
+        prompt,
+        temperature
+      });
+      return {
+        score: result.object.score,
+        reasoning: result.object.reasoning
+      };
+    }
+  };
+}
+async function gradeWithLLM(content, criterion, config = {}) {
+  const grader = createLLMGrader(config);
+  return grader.grade(content, criterion);
+}
+
+// src/utils/metrics.ts
+function calculateWeightedScore(scores) {
+  if (scores.length === 0) return 0;
+  const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
+  if (totalWeight === 0) return 0;
+  const weightedSum = scores.reduce((sum, s) => sum + s.score * s.weight, 0);
+  return Math.round(weightedSum / totalWeight);
+}
+function calculatePercentage(found, expected) {
+  if (expected === 0) return 100;
+  return Math.round(found / expected * 100);
+}
+function average(numbers) {
+  if (numbers.length === 0) return 0;
+  return numbers.reduce((sum, n) => sum + n, 0) / numbers.length;
+}
+function clamp(value, min, max) {
+  return Math.max(min, Math.min(max, value));
+}
+function normalizeScore(score) {
+  return clamp(Math.round(score), 0, 100);
+}
+
+// src/utils/reporters.ts
+function formatConsoleReport(result, options = {}) {
+  const lines = [];
+  const { includeEvidence = true } = options;
+  const status = result.passed ? "\u2713 PASSED" : "\u2717 FAILED";
+  const statusColor = result.passed ? "\x1B[32m" : "\x1B[31m";
+  const reset = "\x1B[0m";
+  lines.push("");
+  lines.push("\u2550".repeat(60));
+  lines.push(`${statusColor}${status}${reset} - Overall Score: ${result.overallScore}/100`);
+  lines.push("\u2550".repeat(60));
+  lines.push("");
+  lines.push(`Summary: ${result.summary}`);
+  lines.push(`Duration: ${result.duration}ms`);
+  lines.push("");
+  lines.push("Scores:");
+  for (const [name, score] of Object.entries(result.scores)) {
+    const bar = createProgressBar(score, 20);
+    const formattedName = formatScoreName(name);
+    lines.push(`  ${formattedName.padEnd(20)} ${bar} ${score}/100`);
+  }
+  if (includeEvidence && result.evidence.length > 0) {
+    lines.push("");
+    lines.push("Evidence:");
+    for (const evidence of result.evidence) {
+      lines.push(`  [${evidence.criterion}] (${evidence.score}/100)`);
+      lines.push(`    ${evidence.reasoning}`);
+      if (evidence.examples && evidence.examples.length > 0) {
+        for (const example of evidence.examples.slice(0, 3)) {
+          lines.push(`    - ${example}`);
+        }
+      }
+    }
+  }
+  lines.push("");
+  lines.push("\u2500".repeat(60));
+  return lines.join("\n");
+}
+function createProgressBar(value, width) {
+  const filled = Math.round(value / 100 * width);
+  const empty = width - filled;
+  return `[${"\u2588".repeat(filled)}${"\u2591".repeat(empty)}]`;
+}
+function formatScoreName(name) {
+  return name.replace(/([A-Z])/g, " $1").replace(/^./, (str) => str.toUpperCase()).trim();
+}
+function formatJsonReport(result, options = {}) {
+  const { includeEvidence = true, includeRawData = false } = options;
+  const output = {
+    passed: result.passed,
+    overallScore: result.overallScore,
+    scores: result.scores,
+    summary: result.summary,
+    duration: result.duration
+  };
+  if (includeEvidence) {
+    output.evidence = result.evidence;
+  }
+  for (const [key, value] of Object.entries(result)) {
+    if (!["passed", "overallScore", "scores", "evidence", "summary", "duration"].includes(key) && (includeRawData || !isRawData(value))) {
+      output[key] = value;
+    }
+  }
+  return JSON.stringify(output, null, 2);
+}
+function isRawData(value) {
+  if (Array.isArray(value) && value.length > 10) return true;
+  if (typeof value === "object" && value !== null) {
+    const keys = Object.keys(value);
+    if (keys.length > 20) return true;
+  }
+  return false;
+}
+function generateSummary(scores, thresholds = { good: 80, acceptable: 60 }) {
+  const entries = Object.entries(scores);
+  const avgScore = entries.reduce((sum, [, score]) => sum + score, 0) / entries.length;
+  const goodMetrics = entries.filter(([, score]) => score >= thresholds.good).map(([name]) => name);
+  const poorMetrics = entries.filter(([, score]) => score < thresholds.acceptable).map(([name]) => name);
+  const parts = [];
+  if (avgScore >= thresholds.good) {
+    parts.push("Strong overall performance");
+  } else if (avgScore >= thresholds.acceptable) {
+    parts.push("Acceptable performance with room for improvement");
+  } else {
+    parts.push("Performance below acceptable thresholds");
+  }
+  if (goodMetrics.length > 0) {
+    parts.push(`Strong: ${goodMetrics.map(formatScoreName).join(", ")}`);
+  }
+  if (poorMetrics.length > 0) {
+    parts.push(`Needs improvement: ${poorMetrics.map(formatScoreName).join(", ")}`);
+  }
+  return parts.join(". ") + ".";
+}
+
+// src/introspection/index.ts
+async function evaluateIntrospection(config) {
+  const startTime = Date.now();
+  const apiKey = config.pineconeApiKey || process.env.PINECONE_API_KEY;
+  if (!apiKey) {
+    throw new Error("PINECONE_API_KEY is required for introspection eval");
+  }
+  const pinecone = new Pinecone({ apiKey });
+  const assistant = pinecone.assistant(config.assistantName);
+  const [entityResult, slotResult, scopeResult, capabilityResult] = await Promise.all([
+    evaluateEntityCoverage(assistant, config.manifest, config),
+    evaluateSlotAccuracy(assistant, config.manifest, config),
+    evaluateScopePrecision(assistant, config.manifest, config),
+    evaluateCapabilityMatch(assistant, config.manifest, config)
+  ]);
+  const scores = {
+    entityCoverage: entityResult.score,
+    slotAccuracy: slotResult.score,
+    scopePrecision: scopeResult.score,
+    capabilityMatch: capabilityResult.score
+  };
+  const overallScore = calculateWeightedScore([
+    { score: scores.entityCoverage, weight: 0.25 },
+    { score: scores.slotAccuracy, weight: 0.3 },
+    { score: scores.scopePrecision, weight: 0.25 },
+    { score: scores.capabilityMatch, weight: 0.2 }
+  ]);
+  const evidence = [
+    ...entityResult.evidence,
+    ...slotResult.evidence,
+    ...scopeResult.evidence,
+    ...capabilityResult.evidence
+  ];
+  return {
+    passed: overallScore >= 70,
+    overallScore,
+    scores,
+    evidence,
+    summary: generateSummary(scores),
+    duration: Date.now() - startTime,
+    details: {
+      missingEntities: entityResult.missing,
+      incorrectSlots: slotResult.incorrect,
+      scopeMisclassifications: scopeResult.misclassified,
+      mismatchedCapabilities: capabilityResult.mismatched
+    },
+    manifest: config.manifest
+  };
+}
+async function evaluateEntityCoverage(assistant, manifest, config) {
+  const grader = createLLMGrader({ openaiApiKey: config.openaiApiKey });
+  const manifestEntities = extractEntitiesFromManifest(manifest);
+  const discoveryQueries = [
+    "What are the main topics you can help with?",
+    "What products or entities do you have information about?",
+    "List the categories of information you contain."
+  ];
+  const discoveredEntities = /* @__PURE__ */ new Set();
+  for (const query of discoveryQueries) {
+    try {
+      const response = await assistant.chat({
+        messages: [{ role: "user", content: query }]
+      });
+      const entities = await extractEntitiesWithLLM(grader, response.message?.content || "");
+      entities.forEach((e) => discoveredEntities.add(e.toLowerCase()));
+    } catch {
+    }
+  }
+  if (config.groundTruth) {
+    for (const gt of config.groundTruth) {
+      if (gt.expectedEntities) {
+        gt.expectedEntities.forEach((e) => discoveredEntities.add(e.toLowerCase()));
+      }
+    }
+  }
+  const found = [];
+  const missing = [];
+  for (const entity of discoveredEntities) {
+    if (manifestEntities.some((me) => me.toLowerCase().includes(entity) || entity.includes(me.toLowerCase()))) {
+      found.push(entity);
+    } else {
+      missing.push(entity);
+    }
+  }
+  const score = discoveredEntities.size === 0 ? 100 : normalizeScore(found.length / discoveredEntities.size * 100);
+  return {
+    score,
+    evidence: [
+      {
+        criterion: "entityCoverage",
+        score,
+        reasoning: `Found ${found.length}/${discoveredEntities.size} expected entities in the manifest.`,
+        examples: missing.slice(0, 5)
+      }
+    ],
+    found,
+    missing
+  };
+}
+async function evaluateSlotAccuracy(assistant, manifest, config) {
+  const grader = createLLMGrader({ openaiApiKey: config.openaiApiKey });
+  const slots = manifest.slots || [];
+  if (slots.length === 0) {
+    return {
+      score: 50,
+      // Penalize but don't fail for no slots
+      evidence: [
+        {
+          criterion: "slotAccuracy",
+          score: 50,
+          reasoning: "No slots defined in manifest. Consider adding slots for common query parameters."
+        }
+      ],
+      correct: [],
+      incorrect: []
+    };
+  }
+  const correct = [];
+  const incorrect = [];
+  for (const slot of slots) {
+    const testQuery = `To answer questions about ${manifest.domain || "this topic"}, do I need to know the ${slot.name}?`;
+    try {
+      const response = await assistant.chat({
+        messages: [{ role: "user", content: testQuery }]
+      });
+      const evaluation = await grader.grade(
+        `Slot: ${slot.name}
+Description: ${slot.description || "N/A"}
+KB Response: ${response.message?.content}`,
+        "slot relevance",
+        "Score 100 if the slot seems relevant to the KB content, 0 if completely irrelevant."
+      );
+      if (evaluation.score >= 60) {
+        correct.push(slot.name);
+      } else {
+        incorrect.push(slot.name);
+      }
+    } catch {
+      correct.push(slot.name);
+    }
+  }
+  const score = normalizeScore(correct.length / slots.length * 100);
+  return {
+    score,
+    evidence: [
+      {
+        criterion: "slotAccuracy",
+        score,
+        reasoning: `${correct.length}/${slots.length} slots appear relevant to the KB content.`,
+        examples: incorrect
+      }
+    ],
+    correct,
+    incorrect
+  };
+}
+async function evaluateScopePrecision(assistant, manifest, config) {
+  const grader = createLLMGrader({ openaiApiKey: config.openaiApiKey });
+  const scope = manifest.scope;
+  if (!scope) {
+    return {
+      score: 60,
+      // Penalize but don't fail
+      evidence: [
+        {
+          criterion: "scopePrecision",
+          score: 60,
+          reasoning: "No scope definition in manifest. Consider defining in-scope and out-of-scope examples."
+        }
+      ],
+      correctClassifications: 0,
+      totalClassifications: 0,
+      misclassified: []
+    };
+  }
+  const testCases = [];
+  if (scope.inScopeExamples) {
+    scope.inScopeExamples.forEach((q) => testCases.push({ query: q, expectedInScope: true }));
+  }
+  if (scope.outOfScopeExamples) {
+    scope.outOfScopeExamples.forEach((q) => testCases.push({ query: q, expectedInScope: false }));
+  }
+  if (config.groundTruth) {
+    config.groundTruth.filter((gt) => gt.shouldBeInScope !== void 0).forEach((gt) => testCases.push({ query: gt.query, expectedInScope: gt.shouldBeInScope }));
+  }
+  if (testCases.length === 0) {
+    return {
+      score: 70,
+      evidence: [
+        {
+          criterion: "scopePrecision",
+          score: 70,
+          reasoning: "No scope test cases available. Add in-scope and out-of-scope examples to test."
+        }
+      ],
+      correctClassifications: 0,
+      totalClassifications: 0,
+      misclassified: []
+    };
+  }
+  let correctClassifications = 0;
+  const misclassified = [];
+  for (const testCase of testCases.slice(0, 10)) {
+    try {
+      const response = await assistant.chat({
+        messages: [{ role: "user", content: testCase.query }]
+      });
+      const evaluation = await grader.grade(
+        `Query: ${testCase.query}
+KB Response: ${response.message?.content}`,
+        "answerability",
+        "Score 100 if the KB provided a substantive, on-topic answer. Score 0 if it said it cannot help or gave an off-topic response."
+      );
+      const actuallyInScope = evaluation.score >= 50;
+      if (actuallyInScope === testCase.expectedInScope) {
+        correctClassifications++;
+      } else {
+        misclassified.push(`"${testCase.query}" (expected ${testCase.expectedInScope ? "in-scope" : "out-of-scope"})`);
+      }
+    } catch {
+    }
+  }
+  const score = normalizeScore(correctClassifications / testCases.length * 100);
+  return {
+    score,
+    evidence: [
+      {
+        criterion: "scopePrecision",
+        score,
+        reasoning: `${correctClassifications}/${testCases.length} scope classifications were correct.`,
+        examples: misclassified.slice(0, 3)
+      }
+    ],
+    correctClassifications,
+    totalClassifications: testCases.length,
+    misclassified
+  };
+}
+async function evaluateCapabilityMatch(assistant, manifest, config) {
+  const grader = createLLMGrader({ openaiApiKey: config.openaiApiKey });
+  const capabilities = (manifest.capabilities || []).map(
+    (c) => typeof c === "string" ? c : c.text
+  );
+  if (capabilities.length === 0) {
+    return {
+      score: 50,
+      evidence: [
+        {
+          criterion: "capabilityMatch",
+          score: 50,
+          reasoning: "No capabilities defined in manifest."
+        }
+      ],
+      matched: [],
+      mismatched: []
+    };
+  }
+  const matched = [];
+  const mismatched = [];
+  for (const capability of capabilities.slice(0, 5)) {
+    const testQuery = `Can you help me with: ${capability}`;
+    try {
+      const response = await assistant.chat({
+        messages: [{ role: "user", content: testQuery }]
+      });
+      const evaluation = await grader.grade(
+        `Capability: ${capability}
+KB Response: ${response.message?.content}`,
+        "capability fulfillment",
+        "Score 100 if the KB demonstrated it can help with this capability. Score 0 if it cannot."
+      );
+      if (evaluation.score >= 60) {
+        matched.push(capability);
+      } else {
+        mismatched.push(capability);
+      }
+    } catch {
+      matched.push(capability);
+    }
+  }
+  const score = normalizeScore(matched.length / capabilities.length * 100);
+  return {
+    score,
+    evidence: [
+      {
+        criterion: "capabilityMatch",
+        score,
+        reasoning: `${matched.length}/${capabilities.length} stated capabilities match actual KB content.`,
+        examples: mismatched
+      }
+    ],
+    matched,
+    mismatched
+  };
+}
+function extractEntitiesFromManifest(manifest) {
+  const entities = /* @__PURE__ */ new Set();
+  if (manifest.slots) {
+    manifest.slots.forEach((slot) => {
+      if (slot.examples) {
+        slot.examples.forEach((e) => entities.add(e));
+      }
+    });
+  }
+  if (manifest.domain) {
+    entities.add(manifest.domain);
+  }
+  if (manifest.capabilities) {
+    manifest.capabilities.forEach((c) => {
+      const text = typeof c === "string" ? c : c.text;
+      const words = text.split(/\s+/).filter((w) => w.length > 3 && /^[A-Z]/.test(w));
+      words.forEach((w) => entities.add(w.toLowerCase()));
+    });
+  }
+  return Array.from(entities);
+}
+async function extractEntitiesWithLLM(grader, text) {
+  try {
+    const result = await grader.grade(
+      text,
+      "entity extraction",
+      "List the main entities (products, topics, categories) mentioned. Return just the entity names separated by commas."
+    );
+    return result.reasoning.split(",").map((e) => e.trim().toLowerCase()).filter((e) => e.length > 2);
+  } catch {
+    return [];
+  }
+}
+async function evaluateRetrieval(config) {
+  const startTime = Date.now();
+  if (config.queries.length === 0) {
+    throw new Error("At least one query is required for retrieval eval");
+  }
+  const apiKey = config.pineconeApiKey || process.env.PINECONE_API_KEY;
+  if (!apiKey) {
+    throw new Error("PINECONE_API_KEY is required for retrieval eval");
+  }
+  const pinecone = new Pinecone({ apiKey });
+  const assistant = pinecone.assistant(config.assistantName);
+  const grader = createLLMGrader({ openaiApiKey: config.openaiApiKey });
+  const topK = config.topK || 5;
+  const queryResults = [];
+  for (const testQuery of config.queries) {
+    const result = await evaluateQueryRetrieval(
+      assistant,
+      grader,
+      testQuery,
+      topK,
+      config.verbose
+    );
+    queryResults.push(result);
+  }
+  const relevanceScores = queryResults.map((r) => r.relevanceScore);
+  const relevance = normalizeScore(average(relevanceScores));
+  const totalExpected = queryResults.reduce(
+    (sum, r) => sum + r.foundTopics.length + r.missingTopics.length,
+    0
+  );
+  const totalFound = queryResults.reduce((sum, r) => sum + r.foundTopics.length, 0);
+  const recall = totalExpected === 0 ? 100 : normalizeScore(totalFound / totalExpected * 100);
+  const allChunks = queryResults.flatMap((r) => r.chunks);
+  const relevantChunks = allChunks.filter((c) => c.relevanceGrade >= 50);
+  const precision = allChunks.length === 0 ? 100 : normalizeScore(relevantChunks.length / allChunks.length * 100);
+  const totalIrrelevant = queryResults.reduce(
+    (sum, r) => sum + r.noiseTopics.length,
+    0
+  );
+  const totalIrrelevantExpected = config.queries.reduce(
+    (sum, q) => sum + (q.irrelevantTopics?.length || 0),
+    0
+  );
+  const noiseRatio = totalIrrelevantExpected === 0 ? 0 : normalizeScore(totalIrrelevant / totalIrrelevantExpected * 100);
+  const scores = { relevance, recall, precision, noiseRatio };
+  const overallScore = calculateWeightedScore([
+    { score: relevance, weight: 0.35 },
+    { score: recall, weight: 0.3 },
+    { score: precision, weight: 0.25 },
+    { score: 100 - noiseRatio, weight: 0.1 }
+    // Invert noise ratio
+  ]);
+  const evidence = [
+    {
+      criterion: "relevance",
+      score: relevance,
+      reasoning: `Average relevance of retrieved chunks across ${queryResults.length} queries.`
+    },
+    {
+      criterion: "recall",
+      score: recall,
+      reasoning: `Found ${totalFound}/${totalExpected} expected topics in retrieved content.`,
+      examples: queryResults.flatMap((r) => r.missingTopics).slice(0, 3)
+    },
+    {
+      criterion: "precision",
+      score: precision,
+      reasoning: `${relevantChunks.length}/${allChunks.length} retrieved chunks were relevant.`
+    },
+    {
+      criterion: "noiseRatio",
+      score: noiseRatio,
+      reasoning: `${totalIrrelevant} irrelevant topics appeared in retrieved content.`,
+      examples: queryResults.flatMap((r) => r.noiseTopics).slice(0, 3)
+    }
+  ];
+  return {
+    passed: overallScore >= 70 && noiseRatio <= 30,
+    overallScore,
+    scores,
+    evidence,
+    summary: generateSummary({ relevance, recall, precision, noiseRatio: 100 - noiseRatio }),
+    duration: Date.now() - startTime,
+    queryResults
+  };
+}
+async function evaluateQueryRetrieval(assistant, grader, testQuery, topK, verbose) {
+  let contextResult;
+  try {
+    contextResult = await assistant.context({
+      query: testQuery.query,
+      topK
+    });
+  } catch (error) {
+    return {
+      query: testQuery.query,
+      chunks: [],
+      relevanceScore: 0,
+      foundTopics: [],
+      missingTopics: testQuery.expectedTopics || [],
+      noiseTopics: []
+    };
+  }
+  const snippets = contextResult.snippets || [];
+  const chunks = [];
+  for (const snippet of snippets) {
+    const relevanceResult = await grader.gradeRelevance(testQuery.query, snippet.content);
+    const ref = snippet.reference;
+    const sourceFile = ref?.file?.name || ref?.name || "unknown";
+    chunks.push({
+      content: snippet.content.slice(0, 500),
+      // Truncate for storage
+      score: snippet.score,
+      sourceFile,
+      relevanceGrade: relevanceResult.score,
+      reasoning: relevanceResult.reasoning
+    });
+  }
+  const relevanceScore = chunks.length === 0 ? 0 : average(chunks.map((c) => c.relevanceGrade));
+  const allContent = chunks.map((c) => c.content).join(" ").toLowerCase();
+  const topicResult = checkTopics(
+    allContent,
+    testQuery.expectedTopics || [],
+    testQuery.irrelevantTopics || []
+  );
+  if (verbose) {
+    console.log(`Query: "${testQuery.query}"`);
+    console.log(`  Relevance: ${relevanceScore.toFixed(1)}/100`);
+    console.log(`  Topics found: ${topicResult.found.join(", ") || "none"}`);
+    console.log(`  Topics missing: ${topicResult.missing.join(", ") || "none"}`);
+    console.log(`  Noise: ${topicResult.noise.join(", ") || "none"}`);
+  }
+  return {
+    query: testQuery.query,
+    chunks,
+    relevanceScore,
+    foundTopics: topicResult.found,
+    missingTopics: topicResult.missing,
+    noiseTopics: topicResult.noise
+  };
+}
+function checkTopics(content, expectedTopics, irrelevantTopics) {
+  const found = [];
+  const missing = [];
+  const noise = [];
+  for (const topic of expectedTopics) {
+    if (content.includes(topic.toLowerCase())) {
+      found.push(topic);
+    } else {
+      missing.push(topic);
+    }
+  }
+  for (const topic of irrelevantTopics) {
+    if (content.includes(topic.toLowerCase())) {
+      noise.push(topic);
+    }
+  }
+  return { found, missing, noise };
+}
+
+// src/agent/index.ts
+async function evaluateAgent(config) {
+  const startTime = Date.now();
+  if (config.scenarios.length === 0) {
+    throw new Error("At least one scenario is required for agent eval");
+  }
+  const grader = createLLMGrader({
+    openaiApiKey: config.openaiApiKey,
+    model: config.graderConfig?.model,
+    temperature: config.graderConfig?.temperature
+  });
+  const scenarioResults = [];
+  for (const scenario of config.scenarios) {
+    const result = await runScenario(scenario, config, grader);
+    scenarioResults.push(result);
+  }
+  const passedScenarios = scenarioResults.filter((r) => r.passed);
+  const accuracy = normalizeScore(passedScenarios.length / scenarioResults.length * 100);
+  const gradedResults = scenarioResults.filter((r) => r.evaluation.evidence.length > 0);
+  const relevanceScores = gradedResults.flatMap(
+    (r) => r.evaluation.evidence.filter((e) => e.criterion === "relevance").map((e) => e.score)
+  );
+  const completenessScores = gradedResults.flatMap(
+    (r) => r.evaluation.evidence.filter((e) => e.criterion === "completeness").map((e) => e.score)
+  );
+  const helpfulnessScores = gradedResults.flatMap(
+    (r) => r.evaluation.evidence.filter((e) => e.criterion === "helpfulness").map((e) => e.score)
+  );
+  const relevance = relevanceScores.length > 0 ? normalizeScore(average(relevanceScores)) : 0;
+  const completeness = completenessScores.length > 0 ? normalizeScore(average(completenessScores)) : 0;
+  const helpfulness = helpfulnessScores.length > 0 ? normalizeScore(average(helpfulnessScores)) : 0;
+  const scores = { accuracy, relevance, completeness, helpfulness };
+  const overallScore = calculateWeightedScore([
+    { score: accuracy, weight: 0.3 },
+    { score: relevance, weight: 0.25 },
+    { score: completeness, weight: 0.25 },
+    { score: helpfulness, weight: 0.2 }
+  ]);
+  const evidence = [
+    {
+      criterion: "accuracy",
+      score: accuracy,
+      reasoning: `${passedScenarios.length}/${scenarioResults.length} scenarios passed.`,
+      examples: scenarioResults.filter((r) => !r.passed).map((r) => r.scenario.name).slice(0, 3)
+    },
+    {
+      criterion: "relevance",
+      score: relevance,
+      reasoning: `Average answer relevance across ${gradedResults.length} graded scenarios.`
+    },
+    {
+      criterion: "completeness",
+      score: completeness,
+      reasoning: `Average answer completeness across ${gradedResults.length} graded scenarios.`
+    },
+    {
+      criterion: "helpfulness",
+      score: helpfulness,
+      reasoning: `Average helpfulness across ${gradedResults.length} graded scenarios.`
+    }
+  ];
+  return {
+    passed: overallScore >= 70,
+    overallScore,
+    scores,
+    evidence,
+    summary: generateSummary(scores),
+    duration: Date.now() - startTime,
+    scenarioResults
+  };
+}
+async function runScenario(scenario, config, grader) {
+  const startTime = Date.now();
+  const maxTurns = scenario.maxTurns || config.maxTurns || 5;
+  const timeout = config.timeout || 6e4;
+  const conversation = [];
+  let currentMessage = scenario.initialQuery;
+  let context = {
+    sessionId: `eval_${Date.now()}`,
+    conversationHistory: []
+  };
+  let lastResponse = null;
+  let turn = 0;
+  try {
+    while (turn < maxTurns) {
+      turn++;
+      const response = await callAgent(config.agentEndpoint, currentMessage, context, timeout);
+      lastResponse = response;
+      conversation.push({
+        turn,
+        userMessage: currentMessage,
+        agentResponse: response
+      });
+      context = {
+        ...context,
+        previousContext: response.context,
+        previousIntent: response.intent,
+        conversationHistory: [
+          ...context.conversationHistory || [],
+          { role: "user", content: currentMessage },
+          { role: "assistant", content: response.answer || response.followUpQuestion || "" }
+        ]
+      };
+      if (response.outcome === "answer") {
+        break;
+      }
+      if (response.outcome === "blocked" || response.outcome === "out_of_scope") {
+        break;
+      }
+      if (response.outcome === "follow_up") {
+        const followUpQuestion = response.followUpQuestion || "";
+        const responseToFollowUp = generateFollowUpResponse(followUpQuestion, scenario);
+        if (!responseToFollowUp) {
+          break;
+        }
+        currentMessage = responseToFollowUp;
+      }
+    }
+    const evaluation = await evaluateScenarioResult(
+      scenario,
+      lastResponse,
+      conversation,
+      grader
+    );
+    const outcomeMatch = scenario.expectedOutcome ? lastResponse?.outcome === scenario.expectedOutcome : true;
+    const passed = outcomeMatch && evaluation.passed;
+    return {
+      scenario,
+      passed,
+      turns: turn,
+      finalOutcome: lastResponse?.outcome || "error",
+      finalAnswer: lastResponse?.outcome === "answer" ? lastResponse.answer || null : null,
+      evaluation,
+      conversation,
+      duration: Date.now() - startTime
+    };
+  } catch (error) {
+    return {
+      scenario,
+      passed: false,
+      turns: turn,
+      finalOutcome: "error",
+      finalAnswer: null,
+      evaluation: {
+        passed: false,
+        score: 0,
+        evidence: [
+          {
+            criterion: "error",
+            score: 0,
+            reasoning: error instanceof Error ? error.message : String(error)
+          }
+        ]
+      },
+      conversation,
+      duration: Date.now() - startTime,
+      error: error instanceof Error ? error.message : String(error)
+    };
+  }
+}
+async function callAgent(endpoint, message, context, timeout) {
+  if (typeof endpoint === "function") {
+    return endpoint(message, context);
+  }
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        message,
+        session_id: context.sessionId,
+        previous_context: context.previousContext,
+        previous_intent: context.previousIntent,
+        conversation_history: context.conversationHistory
+      }),
+      signal: controller.signal
+    });
+    if (!response.ok) {
+      throw new Error(`Agent returned ${response.status}: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return {
+      outcome: data.outcome || "answer",
+      answer: data.answer,
+      followUpQuestion: data.followUpQuestion || data.follow_up_question,
+      options: data.options,
+      context: data.context,
+      intent: data.intent,
+      trace: data.trace,
+      sessionId: data.session_id || data.sessionId
+    };
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
+function generateFollowUpResponse(question, scenario) {
+  if (!scenario.followUpResponses) {
+    return null;
+  }
+  for (const [pattern, response] of Object.entries(scenario.followUpResponses)) {
+    if (question.toLowerCase().includes(pattern.toLowerCase())) {
+      return response;
+    }
+  }
+  return null;
+}
+async function evaluateScenarioResult(scenario, response, conversation, grader) {
+  const evidence = [];
+  if (!response || response.outcome !== "answer" || !response.answer) {
+    if (scenario.expectedOutcome && scenario.expectedOutcome !== "answer") {
+      return {
+        passed: response?.outcome === scenario.expectedOutcome,
+        score: response?.outcome === scenario.expectedOutcome ? 100 : 0,
+        evidence: [
+          {
+            criterion: "outcomeMatch",
+            score: response?.outcome === scenario.expectedOutcome ? 100 : 0,
+            reasoning: `Expected ${scenario.expectedOutcome}, got ${response?.outcome || "no response"}.`
+          }
+        ]
+      };
+    }
+    return {
+      passed: false,
+      score: 0,
+      evidence: [
+        {
+          criterion: "noAnswer",
+          score: 0,
+          reasoning: `Expected an answer but got ${response?.outcome || "no response"}.`
+        }
+      ]
+    };
+  }
+  const answer = response.answer;
+  let totalScore = 0;
+  let criteriaCount = 0;
+  if (scenario.evaluation.mustContain) {
+    const found = scenario.evaluation.mustContain.filter(
+      (s) => answer.toLowerCase().includes(s.toLowerCase())
+    );
+    const score = normalizeScore(found.length / scenario.evaluation.mustContain.length * 100);
+    totalScore += score;
+    criteriaCount++;
+    evidence.push({
+      criterion: "mustContain",
+      score,
+      reasoning: `Found ${found.length}/${scenario.evaluation.mustContain.length} required terms.`,
+      examples: scenario.evaluation.mustContain.filter(
+        (s) => !answer.toLowerCase().includes(s.toLowerCase())
+      )
+    });
+  }
+  if (scenario.evaluation.mustNotContain) {
+    const found = scenario.evaluation.mustNotContain.filter(
+      (s) => answer.toLowerCase().includes(s.toLowerCase())
+    );
+    const score = normalizeScore((scenario.evaluation.mustNotContain.length - found.length) / scenario.evaluation.mustNotContain.length * 100);
+    totalScore += score;
+    criteriaCount++;
+    evidence.push({
+      criterion: "mustNotContain",
+      score,
+      reasoning: `Found ${found.length} forbidden terms.`,
+      examples: found
+    });
+  }
+  const gradingPrompt = `
+Query: ${scenario.initialQuery}
+Answer: ${answer}
+${scenario.evaluation.rubric ? `Rubric: ${scenario.evaluation.rubric}` : ""}
+`;
+  const relevanceResult = await grader.grade(
+    gradingPrompt,
+    "relevance",
+    "How relevant is the answer to the query? 100 = directly and completely addresses the query."
+  );
+  evidence.push({
+    criterion: "relevance",
+    score: relevanceResult.score,
+    reasoning: relevanceResult.reasoning
+  });
+  totalScore += relevanceResult.score;
+  criteriaCount++;
+  const completenessResult = await grader.grade(
+    gradingPrompt,
+    "completeness",
+    "How complete is the answer? 100 = fully addresses all aspects of the query."
+  );
+  evidence.push({
+    criterion: "completeness",
+    score: completenessResult.score,
+    reasoning: completenessResult.reasoning
+  });
+  totalScore += completenessResult.score;
+  criteriaCount++;
+  const helpfulnessResult = await grader.grade(
+    gradingPrompt,
+    "helpfulness",
+    "How helpful and actionable is the answer? 100 = provides clear, actionable guidance."
+  );
+  evidence.push({
+    criterion: "helpfulness",
+    score: helpfulnessResult.score,
+    reasoning: helpfulnessResult.reasoning
+  });
+  totalScore += helpfulnessResult.score;
+  criteriaCount++;
+  const avgScore = criteriaCount > 0 ? totalScore / criteriaCount : 0;
+  return {
+    passed: avgScore >= 70,
+    score: normalizeScore(avgScore),
+    evidence
+  };
+}
+
+export { average, calculatePercentage, calculateWeightedScore, createLLMGrader, evaluateAgent, evaluateIntrospection, evaluateRetrieval, formatConsoleReport, formatJsonReport, gradeWithLLM };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map