evalkit 0.1.0

package/dist/index.js

@@ -0,0 +1,817 @@
+// src/checks/tool-selection.ts
+function toolSelection(input) {
+  const expectedSet = new Set(input.expected);
+  const actualSet = new Set(input.actual);
+  const missing = [...expectedSet].filter((t) => !actualSet.has(t));
+  const extra = [...actualSet].filter((t) => !expectedSet.has(t));
+  const passed = missing.length === 0 && extra.length === 0;
+  let details = passed ? "All expected tools matched" : "";
+  if (missing.length > 0) {
+    details += `Missing: ${missing.join(", ")}`;
+  }
+  if (extra.length > 0) {
+    details += `${missing.length > 0 ? ". " : ""}Unexpected: ${extra.join(", ")}`;
+  }
+  return {
+    key: "tool_selection",
+    passed,
+    details,
+    expected: input.expected,
+    actual: [...actualSet]
+  };
+}
+function createToolSelectionEvaluator(config) {
+  return (input) => toolSelection({ expected: config.expected, actual: input.actual });
+}
+
+// src/checks/content-match.ts
+function contentMatch(input) {
+  if (!input.mustContain || input.mustContain.length === 0) {
+    return {
+      key: "content_match",
+      passed: true,
+      details: "No content requirements",
+      missing: []
+    };
+  }
+  const lower = input.responseText.toLowerCase();
+  const missing = input.mustContain.filter((s) => !lower.includes(s.toLowerCase()));
+  const passed = missing.length === 0;
+  return {
+    key: "content_match",
+    passed,
+    details: passed ? "All required content found" : `Missing: ${missing.join(", ")}`,
+    missing
+  };
+}
+function createContentMatchEvaluator(config) {
+  return (input) => contentMatch({ responseText: input.responseText, mustContain: config.mustContain });
+}
+
+// src/checks/negative-match.ts
+function negativeMatch(input) {
+  if (!input.mustNotContain || input.mustNotContain.length === 0) {
+    return {
+      key: "negative_match",
+      passed: true,
+      details: "No negative requirements",
+      found: []
+    };
+  }
+  const lower = input.responseText.toLowerCase();
+  const found = input.mustNotContain.filter((s) => lower.includes(s.toLowerCase()));
+  const passed = found.length === 0;
+  return {
+    key: "negative_match",
+    passed,
+    details: passed ? "No forbidden content found" : `Found: ${found.join(", ")}`,
+    found
+  };
+}
+function createNegativeMatchEvaluator(config) {
+  return (input) => negativeMatch({ responseText: input.responseText, mustNotContain: config.mustNotContain });
+}
+
+// src/checks/latency.ts
+function latency(input) {
+  const threshold = input.thresholdMs ?? 2e4;
+  const passed = input.latencyMs <= threshold;
+  return {
+    key: "latency",
+    passed,
+    details: passed ? `${input.latencyMs}ms within ${threshold}ms threshold` : `${input.latencyMs}ms exceeded ${threshold}ms threshold`,
+    ms: input.latencyMs,
+    threshold
+  };
+}
+function createLatencyEvaluator(config) {
+  return (input) => latency({ latencyMs: input.latencyMs, thresholdMs: config.thresholdMs });
+}
+
+// src/checks/json-valid.ts
+function jsonValid(input) {
+  let parsed;
+  try {
+    parsed = JSON.parse(input.text);
+  } catch {
+    return {
+      key: "json_valid",
+      passed: false,
+      details: "Invalid JSON"
+    };
+  }
+  if (input.requireObject && (parsed === null || typeof parsed !== "object")) {
+    return {
+      key: "json_valid",
+      passed: false,
+      details: `Expected object or array, got ${parsed === null ? "null" : typeof parsed}`
+    };
+  }
+  return {
+    key: "json_valid",
+    passed: true,
+    details: "Valid JSON"
+  };
+}
+function createJsonValidEvaluator(config) {
+  return (input) => jsonValid({ text: input.text, requireObject: config?.requireObject });
+}
+
+// src/checks/schema-match.ts
+function schemaMatch(input) {
+  const missingKeys = input.requiredKeys.filter((k) => !(k in input.data));
+  const typeErrors = [];
+  if (input.typeChecks) {
+    for (const [key, expectedType] of Object.entries(input.typeChecks)) {
+      if (key in input.data) {
+        const actualType = Array.isArray(input.data[key]) ? "array" : typeof input.data[key];
+        if (actualType !== expectedType) {
+          typeErrors.push(`${key}: expected ${expectedType}, got ${actualType}`);
+        }
+      }
+    }
+  }
+  const passed = missingKeys.length === 0 && typeErrors.length === 0;
+  const problems = [];
+  if (missingKeys.length > 0) problems.push(`Missing keys: ${missingKeys.join(", ")}`);
+  if (typeErrors.length > 0) problems.push(`Type errors: ${typeErrors.join("; ")}`);
+  return {
+    key: "schema_match",
+    passed,
+    details: passed ? "Schema valid" : problems.join(". "),
+    missingKeys,
+    typeErrors
+  };
+}
+function createSchemaMatchEvaluator(config) {
+  return (input) => schemaMatch({
+    data: input.data,
+    requiredKeys: config.requiredKeys,
+    typeChecks: config.typeChecks
+  });
+}
+
+// src/checks/non-empty.ts
+var DEFAULT_COP_OUT_PHRASES = [
+  "i don't know",
+  "n/a",
+  "no information",
+  "i'm not sure",
+  "i cannot",
+  "i can't",
+  "no data available"
+];
+function nonEmpty(input) {
+  const trimmed = input.responseText.trim();
+  if (trimmed.length === 0) {
+    return {
+      key: "non_empty",
+      passed: false,
+      details: "Response is empty"
+    };
+  }
+  const phrases = input.copOutPhrases ?? DEFAULT_COP_OUT_PHRASES;
+  const lower = trimmed.toLowerCase();
+  const matchedPhrase = phrases.find((p) => lower === p.toLowerCase());
+  if (matchedPhrase) {
+    return {
+      key: "non_empty",
+      passed: false,
+      details: `Response is a cop-out phrase: "${matchedPhrase}"`
+    };
+  }
+  return {
+    key: "non_empty",
+    passed: true,
+    details: "Response is non-empty"
+  };
+}
+function createNonEmptyEvaluator(config) {
+  return (input) => nonEmpty({ responseText: input.responseText, copOutPhrases: config?.copOutPhrases });
+}
+
+// src/checks/length-bounds.ts
+function lengthBounds(input) {
+  const length = input.responseText.length;
+  let passed = true;
+  const problems = [];
+  if (input.min !== void 0 && length < input.min) {
+    passed = false;
+    problems.push(`${length} chars below minimum ${input.min}`);
+  }
+  if (input.max !== void 0 && length > input.max) {
+    passed = false;
+    problems.push(`${length} chars above maximum ${input.max}`);
+  }
+  return {
+    key: "length_bounds",
+    passed,
+    details: passed ? `${length} chars within bounds` : problems.join(". "),
+    length,
+    min: input.min,
+    max: input.max
+  };
+}
+function createLengthBoundsEvaluator(config) {
+  return (input) => lengthBounds({ responseText: input.responseText, min: config.min, max: config.max });
+}
+
+// src/checks/regex-match.ts
+function regexMatch(input) {
+  const mode = input.mode ?? "all";
+  const failedPatterns = [];
+  for (const pattern of input.patterns) {
+    const regex = typeof pattern === "string" ? new RegExp(pattern) : pattern;
+    if (!regex.test(input.responseText)) {
+      failedPatterns.push(String(pattern));
+    }
+  }
+  let passed;
+  if (mode === "all") {
+    passed = failedPatterns.length === 0;
+  } else {
+    passed = failedPatterns.length < input.patterns.length;
+  }
+  let details;
+  if (passed) {
+    details = mode === "all" ? "All patterns matched" : "At least one pattern matched";
+  } else {
+    details = mode === "all" ? `Failed patterns: ${failedPatterns.join(", ")}` : "No patterns matched";
+  }
+  return {
+    key: "regex_match",
+    passed,
+    details,
+    failedPatterns
+  };
+}
+function createRegexMatchEvaluator(config) {
+  return (input) => regexMatch({ responseText: input.responseText, patterns: config.patterns, mode: config.mode });
+}
+
+// src/checks/tool-call-count.ts
+function toolCallCount(input) {
+  let passed = true;
+  const problems = [];
+  if (input.min !== void 0 && input.count < input.min) {
+    passed = false;
+    problems.push(`${input.count} calls below minimum ${input.min}`);
+  }
+  if (input.max !== void 0 && input.count > input.max) {
+    passed = false;
+    problems.push(`${input.count} calls above maximum ${input.max}`);
+  }
+  return {
+    key: "tool_call_count",
+    passed,
+    details: passed ? `${input.count} tool calls within bounds` : problems.join(". "),
+    count: input.count,
+    min: input.min,
+    max: input.max
+  };
+}
+function createToolCallCountEvaluator(config) {
+  return (input) => toolCallCount({ count: input.count, min: config.min, max: config.max });
+}
+
+// src/checks/cost-budget.ts
+function costBudget(input) {
+  const passed = input.actual <= input.budget;
+  return {
+    key: "cost_budget",
+    passed,
+    details: passed ? `${input.actual} within budget of ${input.budget}` : `${input.actual} exceeded budget of ${input.budget}`,
+    actual: input.actual,
+    budget: input.budget
+  };
+}
+function createCostBudgetEvaluator(config) {
+  return (input) => costBudget({ actual: input.actual, budget: config.budget });
+}
+
+// src/checks/run-checks.ts
+function runChecks(input) {
+  const results = [];
+  if (input.expectedTools !== void 0 && input.actualTools !== void 0) {
+    results.push(toolSelection({ expected: input.expectedTools, actual: input.actualTools }));
+  }
+  if (input.mustContain !== void 0 && input.responseText !== void 0) {
+    results.push(contentMatch({ responseText: input.responseText, mustContain: input.mustContain }));
+  }
+  if (input.mustNotContain !== void 0 && input.responseText !== void 0) {
+    results.push(negativeMatch({ responseText: input.responseText, mustNotContain: input.mustNotContain }));
+  }
+  if (input.latencyMs !== void 0) {
+    results.push(latency({ latencyMs: input.latencyMs, thresholdMs: input.thresholdMs }));
+  }
+  if (input.json !== void 0) {
+    results.push(jsonValid(input.json));
+  }
+  if (input.schema !== void 0) {
+    results.push(schemaMatch(input.schema));
+  }
+  if (input.copOutPhrases !== void 0 && input.responseText !== void 0) {
+    results.push(nonEmpty({ responseText: input.responseText, copOutPhrases: input.copOutPhrases }));
+  }
+  if ((input.lengthMin !== void 0 || input.lengthMax !== void 0) && input.responseText !== void 0) {
+    results.push(lengthBounds({ responseText: input.responseText, min: input.lengthMin, max: input.lengthMax }));
+  }
+  if (input.regexPatterns !== void 0 && input.responseText !== void 0) {
+    results.push(regexMatch({ responseText: input.responseText, patterns: input.regexPatterns, mode: input.regexMode }));
+  }
+  if (input.toolCallCountValue !== void 0) {
+    results.push(toolCallCount({ count: input.toolCallCountValue, min: input.toolCallMin, max: input.toolCallMax }));
+  }
+  if (input.costActual !== void 0 && input.costBudget !== void 0) {
+    results.push(costBudget({ actual: input.costActual, budget: input.costBudget }));
+  }
+  const passedCount = results.filter((r) => r.passed).length;
+  const allPassed = results.length > 0 && passedCount === results.length;
+  return {
+    passed: allPassed,
+    results,
+    summary: `${passedCount}/${results.length} checks passed`
+  };
+}
+
+// src/runner/loader.ts
+import fs from "fs";
+import path from "path";
+function stripComment(line) {
+  let inSingle = false;
+  let inDouble = false;
+  for (let i = 0; i < line.length; i++) {
+    const ch = line[i];
+    if (ch === "'" && !inDouble) inSingle = !inSingle;
+    else if (ch === '"' && !inSingle) inDouble = !inDouble;
+    else if (ch === "#" && !inSingle && !inDouble) {
+      return line.slice(0, i).trimEnd();
+    }
+  }
+  return line;
+}
+function tokenize(content) {
+  const lines = [];
+  for (const raw of content.split("\n")) {
+    const stripped = stripComment(raw);
+    if (stripped.trim() === "") continue;
+    const indent = stripped.search(/\S/);
+    if (indent === -1) continue;
+    lines.push({ indent, raw: stripped, content: stripped.slice(indent) });
+  }
+  return lines;
+}
+function parseScalar(value) {
+  if (value === "true") return true;
+  if (value === "false") return false;
+  if (value === "null" || value === "~") return null;
+  if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
+    return value.slice(1, -1);
+  }
+  if (/^-?\d+(\.\d+)?$/.test(value)) {
+    return Number(value);
+  }
+  return value;
+}
+function parseFlowArray(raw) {
+  const inner = raw.slice(1, -1).trim();
+  if (inner === "") return [];
+  const items = [];
+  let current = "";
+  let depth = 0;
+  let inSingle = false;
+  let inDouble = false;
+  for (let i = 0; i < inner.length; i++) {
+    const ch = inner[i];
+    if (ch === "'" && !inDouble) {
+      inSingle = !inSingle;
+      current += ch;
+    } else if (ch === '"' && !inSingle) {
+      inDouble = !inDouble;
+      current += ch;
+    } else if (ch === "[" && !inSingle && !inDouble) {
+      depth++;
+      current += ch;
+    } else if (ch === "]" && !inSingle && !inDouble) {
+      depth--;
+      current += ch;
+    } else if (ch === "," && depth === 0 && !inSingle && !inDouble) {
+      items.push(current.trim());
+      current = "";
+    } else {
+      current += ch;
+    }
+  }
+  if (current.trim() !== "") items.push(current.trim());
+  return items.map((item) => parseScalar(item));
+}
+function parseBlock(lines, start, baseIndent) {
+  const result = {};
+  let i = start;
+  while (i < lines.length && lines[i].indent >= baseIndent) {
+    const line = lines[i];
+    if (line.indent < baseIndent) break;
+    if (line.indent > baseIndent) break;
+    const content = line.content;
+    if (content.startsWith("- ")) {
+      break;
+    }
+    const colonIdx = findColon(content);
+    if (colonIdx === -1) {
+      throw new Error(`YAML parse error: expected key-value pair, got "${content}"`);
+    }
+    const key = content.slice(0, colonIdx).trim();
+    const valueRaw = content.slice(colonIdx + 1).trim();
+    if (valueRaw === "" || valueRaw === "") {
+      if (i + 1 < lines.length && lines[i + 1].indent > baseIndent) {
+        const nextIndent = lines[i + 1].indent;
+        const nextContent = lines[i + 1].content;
+        if (nextContent.startsWith("- ")) {
+          const [arr, newI] = parseSequence(lines, i + 1, nextIndent);
+          result[key] = arr;
+          i = newI;
+        } else {
+          const [nested, newI] = parseBlock(lines, i + 1, nextIndent);
+          result[key] = nested;
+          i = newI;
+        }
+      } else {
+        result[key] = null;
+        i++;
+      }
+    } else if (valueRaw.startsWith("[") && valueRaw.endsWith("]")) {
+      result[key] = parseFlowArray(valueRaw);
+      i++;
+    } else {
+      result[key] = parseScalar(valueRaw);
+      i++;
+    }
+  }
+  return [result, i];
+}
+function parseSequence(lines, start, baseIndent) {
+  const result = [];
+  let i = start;
+  while (i < lines.length && lines[i].indent >= baseIndent) {
+    const line = lines[i];
+    if (line.indent < baseIndent) break;
+    if (line.indent > baseIndent) break;
+    const content = line.content;
+    if (!content.startsWith("- ")) break;
+    const afterDash = content.slice(2).trim();
+    if (afterDash === "") {
+      if (i + 1 < lines.length && lines[i + 1].indent > baseIndent) {
+        const nextIndent = lines[i + 1].indent;
+        const [nested, newI] = parseBlock(lines, i + 1, nextIndent);
+        result.push(nested);
+        i = newI;
+      } else {
+        result.push(null);
+        i++;
+      }
+    } else if (findColon(afterDash) !== -1 && !afterDash.startsWith('"') && !afterDash.startsWith("'")) {
+      const inlineIndent = line.indent + 2;
+      const colonIdx = findColon(afterDash);
+      const firstKey = afterDash.slice(0, colonIdx).trim();
+      const firstValRaw = afterDash.slice(colonIdx + 1).trim();
+      const obj = {};
+      if (firstValRaw === "") {
+        if (i + 1 < lines.length && lines[i + 1].indent > inlineIndent) {
+          const nextIndent = lines[i + 1].indent;
+          const nextContent = lines[i + 1].content;
+          if (nextContent.startsWith("- ")) {
+            const [arr, newI] = parseSequence(lines, i + 1, nextIndent);
+            obj[firstKey] = arr;
+            i = newI;
+          } else {
+            const [nested, newI] = parseBlock(lines, i + 1, nextIndent);
+            obj[firstKey] = nested;
+            i = newI;
+          }
+        } else {
+          obj[firstKey] = null;
+          i++;
+        }
+      } else if (firstValRaw.startsWith("[") && firstValRaw.endsWith("]")) {
+        obj[firstKey] = parseFlowArray(firstValRaw);
+        i++;
+      } else {
+        obj[firstKey] = parseScalar(firstValRaw);
+        i++;
+      }
+      while (i < lines.length && lines[i].indent === inlineIndent) {
+        const sibContent = lines[i].content;
+        if (sibContent.startsWith("- ")) break;
+        const sibColon = findColon(sibContent);
+        if (sibColon === -1) break;
+        const sibKey = sibContent.slice(0, sibColon).trim();
+        const sibValRaw = sibContent.slice(sibColon + 1).trim();
+        if (sibValRaw === "") {
+          if (i + 1 < lines.length && lines[i + 1].indent > inlineIndent) {
+            const nextIndent = lines[i + 1].indent;
+            const nextContent = lines[i + 1].content;
+            if (nextContent.startsWith("- ")) {
+              const [arr, newI] = parseSequence(lines, i + 1, nextIndent);
+              obj[sibKey] = arr;
+              i = newI;
+            } else {
+              const [nested, newI] = parseBlock(lines, i + 1, nextIndent);
+              obj[sibKey] = nested;
+              i = newI;
+            }
+          } else {
+            obj[sibKey] = null;
+            i++;
+          }
+        } else if (sibValRaw.startsWith("[") && sibValRaw.endsWith("]")) {
+          obj[sibKey] = parseFlowArray(sibValRaw);
+          i++;
+        } else {
+          obj[sibKey] = parseScalar(sibValRaw);
+          i++;
+        }
+      }
+      result.push(obj);
+    } else if (afterDash.startsWith("[") && afterDash.endsWith("]")) {
+      result.push(parseFlowArray(afterDash));
+      i++;
+    } else {
+      result.push(parseScalar(afterDash));
+      i++;
+    }
+  }
+  return [result, i];
+}
+function findColon(str) {
+  let inSingle = false;
+  let inDouble = false;
+  for (let i = 0; i < str.length; i++) {
+    const ch = str[i];
+    if (ch === "'" && !inDouble) inSingle = !inSingle;
+    else if (ch === '"' && !inSingle) inDouble = !inDouble;
+    else if (ch === ":" && !inSingle && !inDouble) {
+      if (i + 1 >= str.length || str[i + 1] === " ") {
+        return i;
+      }
+    }
+  }
+  return -1;
+}
+function parseYaml(content) {
+  const lines = tokenize(content);
+  if (lines.length === 0) {
+    throw new Error("YAML is empty");
+  }
+  const [parsed] = parseBlock(lines, 0, lines[0].indent);
+  return validate(parsed);
+}
+function parseJson(content) {
+  const parsed = JSON.parse(content);
+  return validate(parsed);
+}
+function loadFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  if (ext !== ".json" && ext !== ".yaml" && ext !== ".yml") {
+    throw new Error(`Unsupported file extension: ${ext}. Use .json, .yaml, or .yml`);
+  }
+  const content = fs.readFileSync(filePath, "utf-8");
+  if (ext === ".json") {
+    return parseJson(content);
+  }
+  return parseYaml(content);
+}
+function loadCases(source) {
+  if (typeof source === "string") {
+    return loadFile(source);
+  }
+  return validate(source);
+}
+function validate(data) {
+  if (typeof data !== "object" || data === null) {
+    throw new Error("Invalid suite config: expected an object");
+  }
+  const obj = data;
+  if (!Array.isArray(obj.test_cases)) {
+    throw new Error("Invalid suite config: missing test_cases array");
+  }
+  for (const tc of obj.test_cases) {
+    if (typeof tc !== "object" || tc === null) {
+      throw new Error("Invalid test case: expected an object");
+    }
+    const c = tc;
+    if (typeof c.id !== "string") {
+      throw new Error("Invalid test case: missing or non-string id");
+    }
+    if (typeof c.query !== "string") {
+      throw new Error("Invalid test case: missing or non-string query");
+    }
+  }
+  return {
+    name: typeof obj.name === "string" ? obj.name : void 0,
+    test_cases: obj.test_cases
+  };
+}
+
+// src/runner/run-suite.ts
+async function runSuite(options) {
+  const config = typeof options.cases === "string" ? loadCases(options.cases) : loadCases(options.cases);
+  const suiteName = options.name ?? config.name ?? "unnamed";
+  const concurrency = options.concurrency ?? 1;
+  const startTime = Date.now();
+  const cases = config.test_cases;
+  const results = [];
+  if (concurrency <= 1) {
+    for (const tc of cases) {
+      const result = await runCase(tc, options.agent);
+      results.push(result);
+      options.onCaseComplete?.(result);
+    }
+  } else {
+    let idx = 0;
+    const runNext = async () => {
+      while (idx < cases.length) {
+        const currentIdx = idx++;
+        const result = await runCase(cases[currentIdx], options.agent);
+        results.push(result);
+        options.onCaseComplete?.(result);
+      }
+    };
+    const workers = Array.from(
+      { length: Math.min(concurrency, cases.length) },
+      () => runNext()
+    );
+    await Promise.all(workers);
+  }
+  const passedCount = results.filter((r) => r.passed).length;
+  const duration = Date.now() - startTime;
+  return {
+    name: suiteName,
+    passed: passedCount,
+    failed: results.length - passedCount,
+    total: results.length,
+    cases: results,
+    duration
+  };
+}
+async function runCase(tc, agent) {
+  let agentResult;
+  try {
+    agentResult = await agent(tc.query);
+  } catch (err) {
+    agentResult = {
+      responseText: ""
+    };
+    return {
+      id: tc.id,
+      query: tc.query,
+      passed: false,
+      checks: {
+        passed: false,
+        results: [{
+          key: "agent_error",
+          passed: false,
+          details: `Agent threw: ${err instanceof Error ? err.message : String(err)}`
+        }],
+        summary: "0/1 checks passed"
+      },
+      metadata: tc.metadata,
+      agentResult
+    };
+  }
+  const checks = tc.checks ?? {};
+  const input = {};
+  input.responseText = agentResult.responseText;
+  if (checks.expectedTools !== void 0) {
+    input.expectedTools = checks.expectedTools;
+    input.actualTools = agentResult.actualTools;
+  }
+  if (checks.mustContain !== void 0) {
+    input.mustContain = checks.mustContain;
+  }
+  if (checks.mustNotContain !== void 0) {
+    input.mustNotContain = checks.mustNotContain;
+  }
+  if (agentResult.latencyMs !== void 0) {
+    input.latencyMs = agentResult.latencyMs;
+  }
+  if (checks.thresholdMs !== void 0) {
+    input.thresholdMs = checks.thresholdMs;
+  }
+  if (checks.json !== void 0) {
+    input.json = {
+      text: agentResult.responseText,
+      requireObject: checks.json.requireObject
+    };
+  }
+  if (checks.schema !== void 0) {
+    try {
+      const data = JSON.parse(agentResult.responseText);
+      input.schema = {
+        data,
+        requiredKeys: checks.schema.requiredKeys,
+        typeChecks: checks.schema.typeChecks
+      };
+    } catch {
+      input.schema = {
+        data: {},
+        requiredKeys: checks.schema.requiredKeys,
+        typeChecks: checks.schema.typeChecks
+      };
+    }
+  }
+  if (checks.copOutPhrases !== void 0) {
+    input.copOutPhrases = checks.copOutPhrases;
+  }
+  if (checks.lengthMin !== void 0) {
+    input.lengthMin = checks.lengthMin;
+  }
+  if (checks.lengthMax !== void 0) {
+    input.lengthMax = checks.lengthMax;
+  }
+  if (checks.regexPatterns !== void 0) {
+    input.regexPatterns = checks.regexPatterns.map((p) => new RegExp(p));
+  }
+  if (checks.regexMode !== void 0) {
+    input.regexMode = checks.regexMode;
+  }
+  const toolCallCountValue = agentResult.toolCallCount ?? agentResult.actualTools?.length;
+  if (toolCallCountValue !== void 0) {
+    input.toolCallCountValue = toolCallCountValue;
+  }
+  if (checks.toolCallMin !== void 0) {
+    input.toolCallMin = checks.toolCallMin;
+  }
+  if (checks.toolCallMax !== void 0) {
+    input.toolCallMax = checks.toolCallMax;
+  }
+  if (agentResult.cost !== void 0) {
+    input.costActual = agentResult.cost;
+  }
+  if (checks.costBudget !== void 0) {
+    input.costBudget = checks.costBudget;
+  }
+  const suiteResult = runChecks(input);
+  const passed = suiteResult.results.length === 0 ? true : suiteResult.passed;
+  return {
+    id: tc.id,
+    query: tc.query,
+    passed,
+    checks: { ...suiteResult, passed },
+    metadata: tc.metadata,
+    agentResult
+  };
+}
+
+// src/runner/reporter.ts
+function printSuiteResult(result) {
+  console.log(`
+Suite: ${result.name}`);
+  console.log(`${"=".repeat(60)}`);
+  for (const c of result.cases) {
+    const status = c.passed ? "PASS" : "FAIL";
+    const query = c.query.length > 50 ? c.query.slice(0, 47) + "..." : c.query;
+    const latency2 = c.agentResult.latencyMs !== void 0 ? `${(c.agentResult.latencyMs / 1e3).toFixed(1)}s` : "";
+    console.log(`  ${c.id}  ${query.padEnd(50)}  ${status}  ${latency2}`);
+    if (!c.passed) {
+      for (const check of c.checks.results) {
+        if (!check.passed) {
+          console.log(`           ${check.key}: ${check.details}`);
+        }
+      }
+    }
+  }
+  console.log(`
+${result.passed}/${result.total} passed (${(result.duration / 1e3).toFixed(1)}s)`);
+}
+export {
+  contentMatch,
+  costBudget,
+  createContentMatchEvaluator,
+  createCostBudgetEvaluator,
+  createJsonValidEvaluator,
+  createLatencyEvaluator,
+  createLengthBoundsEvaluator,
+  createNegativeMatchEvaluator,
+  createNonEmptyEvaluator,
+  createRegexMatchEvaluator,
+  createSchemaMatchEvaluator,
+  createToolCallCountEvaluator,
+  createToolSelectionEvaluator,
+  jsonValid,
+  latency,
+  lengthBounds,
+  loadCases,
+  loadFile,
+  negativeMatch,
+  nonEmpty,
+  parseJson,
+  parseYaml,
+  printSuiteResult,
+  regexMatch,
+  runChecks,
+  runSuite,
+  schemaMatch,
+  toolCallCount,
+  toolSelection
+};
+//# sourceMappingURL=index.js.map