evalsense 0.2.0

package/dist/metrics/index.js

@@ -0,0 +1,191 @@
+export { batchItems, createJSONSchema, createLLMError, extractScore, faithfulness, fillPrompt, getLLMClient, hallucination, parseJSONResponse, relevance, requireLLMClient, resetLLMClient, setLLMClient, toxicity, validateResponse, withTimeout } from '../chunk-BRPM6AB6.js';
+
+// src/metrics/custom/index.ts
+var customMetrics = /* @__PURE__ */ new Map();
+function registerMetric(name, fn) {
+  if (customMetrics.has(name)) {
+    throw new Error(`Metric "${name}" is already registered`);
+  }
+  customMetrics.set(name, fn);
+}
+function getMetric(name) {
+  return customMetrics.get(name);
+}
+async function runMetric(name, config) {
+  const fn = customMetrics.get(name);
+  if (!fn) {
+    throw new Error(`Metric "${name}" is not registered`);
+  }
+  return fn(config);
+}
+function listMetrics() {
+  return Array.from(customMetrics.keys());
+}
+function unregisterMetric(name) {
+  return customMetrics.delete(name);
+}
+function clearMetrics() {
+  customMetrics.clear();
+}
+function createPatternMetric(name, patterns, options = {}) {
+  const { matchScore = 1, noMatchScore = 0 } = options;
+  return async (config) => {
+    return config.outputs.map((o) => {
+      const hasMatch = patterns.some((p) => p.test(o.output));
+      return {
+        id: o.id,
+        metric: name,
+        score: hasMatch ? matchScore : noMatchScore,
+        label: hasMatch ? "detected" : "not_detected"
+      };
+    });
+  };
+}
+function createKeywordMetric(name, keywords, options = {}) {
+  const { caseSensitive = false, threshold = 0.5 } = options;
+  const normalizedKeywords = caseSensitive ? keywords : keywords.map((k) => k.toLowerCase());
+  return async (config) => {
+    return config.outputs.map((o) => {
+      const text = caseSensitive ? o.output : o.output.toLowerCase();
+      const matches = normalizedKeywords.filter((k) => text.includes(k));
+      const score = matches.length / keywords.length;
+      return {
+        id: o.id,
+        metric: name,
+        score,
+        label: score >= threshold ? "detected" : "not_detected"
+      };
+    });
+  };
+}
+
+// src/metrics/utils/index.ts
+function normalizeScore(score, min = 0, max = 1) {
+  const range = max - min;
+  if (range === 0) return 0;
+  return Math.max(0, Math.min(1, (score - min) / range));
+}
+function scoreToLabel(score, thresholds) {
+  const sorted = [...thresholds].sort((a, b) => b.min - a.min);
+  for (const { label, min } of sorted) {
+    if (score >= min) {
+      return label;
+    }
+  }
+  return thresholds[thresholds.length - 1]?.label ?? "unknown";
+}
+function createMetricOutput(id, metric, score, labelThresholds) {
+  const normalizedScore = normalizeScore(score);
+  const label = labelThresholds ? scoreToLabel(normalizedScore, labelThresholds) : normalizedScore >= 0.5 ? "high" : "low";
+  return {
+    id,
+    metric,
+    score: normalizedScore,
+    label
+  };
+}
+var BINARY_THRESHOLDS = [
+  { label: "true", min: 0.5 },
+  { label: "false", min: 0 }
+];
+var SEVERITY_THRESHOLDS = [
+  { label: "high", min: 0.7 },
+  { label: "medium", min: 0.4 },
+  { label: "low", min: 0 }
+];
+function batch(items, size) {
+  const batches = [];
+  for (let i = 0; i < items.length; i += size) {
+    batches.push(items.slice(i, i + size));
+  }
+  return batches;
+}
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/metrics/llm/adapters/mock.ts
+function createMockLLMClient(config = {}) {
+  const {
+    response,
+    responses,
+    delay: delay2 = 0,
+    shouldError = false,
+    errorMessage = "Mock LLM error",
+    onPrompt
+  } = config;
+  let callCount = 0;
+  const getResponse = () => {
+    if (responses && responses.length > 0) {
+      const resp = responses[Math.min(callCount, responses.length - 1)];
+      if (!resp) {
+        return JSON.stringify({ score: 0.5, reasoning: "Mock response" });
+      }
+      callCount++;
+      return resp;
+    }
+    if (response !== void 0) {
+      return response;
+    }
+    return JSON.stringify({ score: 0.5, reasoning: "Mock response" });
+  };
+  return {
+    async complete(prompt) {
+      if (onPrompt) {
+        onPrompt(prompt);
+      }
+      if (delay2 > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delay2));
+      }
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      const resp = getResponse();
+      return typeof resp === "string" ? resp : JSON.stringify(resp);
+    },
+    async completeStructured(prompt, _schema) {
+      if (onPrompt) {
+        onPrompt(prompt);
+      }
+      if (delay2 > 0) {
+        await new Promise((resolve) => setTimeout(resolve, delay2));
+      }
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      const resp = getResponse();
+      if (typeof resp === "string") {
+        try {
+          return JSON.parse(resp);
+        } catch {
+          throw new Error(`Mock response is not valid JSON: ${resp}`);
+        }
+      }
+      return resp;
+    }
+  };
+}
+function createSequentialMockClient(responses, options = {}) {
+  return createMockLLMClient({
+    responses,
+    delay: options.delay
+  });
+}
+function createErrorMockClient(errorMessage = "Mock LLM error") {
+  return createMockLLMClient({
+    shouldError: true,
+    errorMessage
+  });
+}
+function createSpyMockClient(response) {
+  const prompts = [];
+  const client = createMockLLMClient({
+    response,
+    onPrompt: (prompt) => prompts.push(prompt)
+  });
+  return { client, prompts };
+}
+
+export { BINARY_THRESHOLDS, SEVERITY_THRESHOLDS, batch, clearMetrics, createErrorMockClient, createKeywordMetric, createMetricOutput, createMockLLMClient, createPatternMetric, createSequentialMockClient, createSpyMockClient, delay, getMetric, listMetrics, normalizeScore, registerMetric, runMetric, scoreToLabel, unregisterMetric };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/metrics/custom/index.ts","../../src/metrics/utils/index.ts","../../src/metrics/llm/adapters/mock.ts"],"names":["delay"],"mappings":";;;AASA,IAAM,aAAA,uBAAoB,GAAA,EAAsB;AAiBzC,SAAS,cAAA,CAAe,MAAc,EAAA,EAAoB;AAC/D,EAAA,IAAI,aAAA,CAAc,GAAA,CAAI,IAAI,CAAA,EAAG;AAC3B,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,QAAA,EAAW,IAAI,CAAA,uBAAA,CAAyB,CAAA;AAAA,EAC1D;AACA,EAAA,aAAA,CAAc,GAAA,CAAI,MAAM,EAAE,CAAA;AAC5B;AAKO,SAAS,UAAU,IAAA,EAAoC;AAC5D,EAAA,OAAO,aAAA,CAAc,IAAI,IAAI,CAAA;AAC/B;AAKA,eAAsB,SAAA,CAAU,MAAc,MAAA,EAA+C;AAC3F,EAAA,MAAM,EAAA,GAAK,aAAA,CAAc,GAAA,CAAI,IAAI,CAAA;AACjC,EAAA,IAAI,CAAC,EAAA,EAAI;AACP,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,QAAA,EAAW,IAAI,CAAA,mBAAA,CAAqB,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,GAAG,MAAM,CAAA;AAClB;AAKO,SAAS,WAAA,GAAwB;AACtC,EAAA,OAAO,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,IAAA,EAAM,CAAA;AACxC;AAKO,SAAS,iBAAiB,IAAA,EAAuB;AACtD,EAAA,OAAO,aAAA,CAAc,OAAO,IAAI,CAAA;AAClC;AAKO,SAAS,YAAA,GAAqB;AACnC,EAAA,aAAA,CAAc,KAAA,EAAM;AACtB;AAKO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA0D,EAAC,EACjD;AACV,EAAA,MAAM,EAAE,UAAA,GAAa,CAAA,EAAG,YAAA,GAAe,GAAE,GAAI,OAAA;AAE7C,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,QAAA,GAAW,SAAS,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,IAAA,CAAK,CAAA,CAAE,MAAM,CAAC,CAAA;AACtD,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA,EAAO,WAAW,UAAA,GAAa,YAAA;AAAA,QAC/B,KAAA,EAAO,WAAW,UAAA,GAAa;AAAA,OACjC;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;AAKO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA2D,EAAC,EAClD;AACV,EAAA,MAAM,EAAE,aAAA,GAAgB,KAAA,EAAO,SAAA,GAAY,KAAI,GAAI,OAAA;AAEnD,EAAA,MAAM,kBAAA,GAAqB,gBACvB,QAAA,GACA,QAAA,CAAS,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,WAAA,EAAa,CAAA;AAEvC,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,OAAO,aAAA,GAAgB,CAAA,CAAE,MAAA,GAAS,CAAA,CAAE,OAAO,WAAA,EAAY;AAC7D,MAAA,MAAM,OAAA,GAAU,mBAAmB,MAAA,CAAO,CAAC,MAAM,IAAA,CAAK,QAAA,CAAS,CAAC,CAAC,CAAA;AACjE,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,GAAS,QAAA,CAAS,MAAA;AAExC,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA;AAAA,QACA,KAAA,EAAO,KAAA,IAAS,SAAA,GAAY,UAAA,GAAa;AAAA,OAC3C;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;;;AClHO,SAAS,cAAA,CAAe,KAAA,EAAe,GAAA,GAAM,CAAA,EAAG,MAAM,CAAA,EAAW;AACtE,EAAA,MAAM,QAAQ,GAAA,GAAM,GAAA;AACpB,EAAA,IAAI,KAAA,KAAU,GAAG,OAAO,CAAA;AACxB,EAAA,OAAO,IAAA,CAAK,IAAI,CAAA,EAAG,IAAA,CAAK,IAAI,CAAA,EAAA,CAAI,KAAA,GAAQ,GAAA,IAAO,KAAK,CAAC,CAAA;AACvD;AAKO,SAAS,YAAA,CACd,OACA,UAAA,EACQ;AAER,EAAA,MAAM,MAAA,GAAS,CAAC,GAAG,UAAU,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA;AAE3D,EAAA,KAAA,MAAW,EAAE,KAAA,EAAO,GAAA,EAAI,IAAK,MAAA,EAAQ;AACnC,IAAA,IAAI,SAAS,GAAA,EAAK;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,UAAA,CAAW,UAAA,CAAW,MAAA,GAAS,CAAC,GAAG,KAAA,IAAS,SAAA;AACrD;AAKO,SAAS,kBAAA,CACd,EAAA,EACA,MAAA,EACA,KAAA,EACA,eAAA,EACc;AACd,EAAA,MAAM,eAAA,GAAkB,eAAe,KAAK,CAAA;AAC5C,EAAA,MAAM,KAAA,GAAQ,kBACV,YAAA,CAAa,eAAA,EAAiB,eAAe,CAAA,GAC7C,eAAA,IAAmB,MACjB,MAAA,GACA,KAAA;AAEN,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,MAAA;AAAA,IACA,KAAA,EAAO,eAAA;AAAA,IACP;AAAA,GACF;AACF;AAKO,IAAM,iBAAA,GAAoB;AAAA,EAC/B,EAAE,KAAA,EAAO,MAAA,EAAQ,GAAA,EAAK,GAAA,EAAI;AAAA,EAC1B,EAAE,KAAA,EAAO,OAAA,EAAS,GAAA,EAAK,CAAA;AACzB;AAKO,IAAM,mBAAA,GAAsB;AAAA,EACjC,EAAE,KAAA,EAAO,MAAA,EAAQ,GAAA,EAAK,GAAA,EAAI;AAAA,EAC1B,EAAE,KAAA,EAAO,QAAA,EAAU,GAAA,EAAK,GAAA,EAAI;AAAA,EAC5B,EAAE,KAAA,EAAO,KAAA,EAAO,GAAA,EAAK,CAAA;AACvB;AAKO,SAAS,KAAA,CAAS,OAAY,IAAA,EAAqB;AACxD,EAAA,MAAM,UAAiB,EAAC;AACxB,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,KAAA,CAAM,MAAA,EAAQ,KAAK,IAAA,EAAM;AAC3C,IAAA,OAAA,CAAQ,KAAK,KAAA,CAAM,KAAA,CAAM,CAAA,EAAG,CAAA,GAAI,IAAI,CAAC,CAAA;AAAA,EACvC;AACA,EAAA,OAAO,OAAA;AACT;AAKO,SAAS,MAAM,EAAA,EAA2B;AAC/C,EAAA,OAAO,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAAS,EAAE,CAAC,CAAA;AACzD;;;AC/CO,SAAS,mBAAA,CAAoB,MAAA,GAAwB,EAAC,EAAc;AACzE,EAAA,MAAM;AAAA,IACJ,QAAA;AAAA,IACA,SAAA;AAAA,IACA,OAAAA,MAAAA,GAAQ,CAAA;AAAA,IACR,WAAA,GAAc,KAAA;AAAA,IACd,YAAA,GAAe,gBAAA;AAAA,IACf;AAAA,GACF,GAAI,MAAA;AAEJ,EAAA,IAAI,SAAA,GAAY,CAAA;AAEhB,EAAA,MAAM,cAAc,MAAwC;AAC1D,IAAA,IAAI,SAAA,IAAa,SAAA,CAAU,MAAA,GAAS,CAAA,EAAG;AACrC,MAAA,MAAM,IAAA,GAAO,UAAU,IAAA,CAAK,GAAA,CAAI,WAAW,SAAA,CAAU,MAAA,GAAS,CAAC,CAAC,CAAA;AAChE,MAAA,IAAI,CAAC,IAAA,EAAM;AACT,QAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,MAClE;AACA,MAAA,SAAA,EAAA;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAI,aAAa,MAAA,EAAW;AAC1B,MAAA,OAAO,QAAA;AAAA,IACT;AAGA,IAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,EAClE,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAE9C,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,OAAO,OAAO,IAAA,KAAS,QAAA,GAAW,IAAA,GAAO,IAAA,CAAK,UAAU,IAAI,CAAA;AAAA,IAC9D,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,OAAA,EAAiC;AAE3E,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,IAAI,OAAO,SAAS,QAAA,EAAU;AAC5B,QAAA,IAAI;AACF,UAAA,OAAO,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,QACxB,CAAA,CAAA,MAAQ;AACN,UAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iCAAA,EAAoC,IAAI,CAAA,CAAE,CAAA;AAAA,QAC5D;AAAA,MACF;AAEA,MAAA,OAAO,IAAA;AAAA,IACT;AAAA,GACF;AACF;AAeO,SAAS,0BAAA,CACd,SAAA,EACA,OAAA,GAA8B,EAAC,EACpB;AACX,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,SAAA;AAAA,IACA,OAAO,OAAA,CAAQ;AAAA,GAChB,CAAA;AACH;AAOO,SAAS,qBAAA,CAAsB,eAAe,gBAAA,EAA6B;AAChF,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,WAAA,EAAa,IAAA;AAAA,IACb;AAAA,GACD,CAAA;AACH;AAcO,SAAS,oBACd,QAAA,EAC0C;AAC1C,EAAA,MAAM,UAAoB,EAAC;AAE3B,EAAA,MAAM,SAAS,mBAAA,CAAoB;AAAA,IACjC,QAAA;AAAA,IACA,QAAA,EAAU,CAAC,MAAA,KAAW,OAAA,CAAQ,KAAK,MAAM;AAAA,GAC1C,CAAA;AAED,EAAA,OAAO,EAAE,QAAQ,OAAA,EAAQ;AAC3B","file":"index.js","sourcesContent":["/**\n * Custom metric registration\n */\n\nimport type { MetricFn, MetricOutput, MetricConfig } from \"../../core/types.js\";\n\n/**\n * Registry of custom metrics\n */\nconst customMetrics = new Map<string, MetricFn>();\n\n/**\n * Registers a custom metric\n *\n * @example\n * ```ts\n * registerMetric(\"custom-relevance\", async ({ outputs, query }) => {\n *   // Custom evaluation logic\n *   return outputs.map(o => ({\n *     id: o.id,\n *     metric: \"custom-relevance\",\n *     score: evaluateRelevance(o.output, query),\n *   }));\n * });\n * ```\n */\nexport function registerMetric(name: string, fn: MetricFn): void {\n  if (customMetrics.has(name)) {\n    throw new Error(`Metric \"${name}\" is already registered`);\n  }\n  customMetrics.set(name, fn);\n}\n\n/**\n * Gets a registered custom metric\n */\nexport function getMetric(name: string): MetricFn | undefined {\n  return customMetrics.get(name);\n}\n\n/**\n * Runs a registered metric\n */\nexport async function runMetric(name: string, config: MetricConfig): Promise<MetricOutput[]> {\n  const fn = customMetrics.get(name);\n  if (!fn) {\n    throw new Error(`Metric \"${name}\" is not registered`);\n  }\n  return fn(config);\n}\n\n/**\n * Lists all registered custom metrics\n */\nexport function listMetrics(): string[] {\n  return Array.from(customMetrics.keys());\n}\n\n/**\n * Unregisters a metric (mainly for testing)\n */\nexport function unregisterMetric(name: string): boolean {\n  return customMetrics.delete(name);\n}\n\n/**\n * Clears all registered metrics (mainly for testing)\n */\nexport function clearMetrics(): void {\n  customMetrics.clear();\n}\n\n/**\n * Creates a simple string-matching metric\n */\nexport function createPatternMetric(\n  name: string,\n  patterns: RegExp[],\n  options: { matchScore?: number; noMatchScore?: number } = {}\n): MetricFn {\n  const { matchScore = 1, noMatchScore = 0 } = options;\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const hasMatch = patterns.some((p) => p.test(o.output));\n      return {\n        id: o.id,\n        metric: name,\n        score: hasMatch ? matchScore : noMatchScore,\n        label: hasMatch ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n\n/**\n * Creates a keyword-based metric\n */\nexport function createKeywordMetric(\n  name: string,\n  keywords: string[],\n  options: { caseSensitive?: boolean; threshold?: number } = {}\n): MetricFn {\n  const { caseSensitive = false, threshold = 0.5 } = options;\n\n  const normalizedKeywords = caseSensitive\n    ? keywords\n    : keywords.map((k) => k.toLowerCase());\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const text = caseSensitive ? o.output : o.output.toLowerCase();\n      const matches = normalizedKeywords.filter((k) => text.includes(k));\n      const score = matches.length / keywords.length;\n\n      return {\n        id: o.id,\n        metric: name,\n        score,\n        label: score >= threshold ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n","/**\n * Metric utilities\n */\n\nimport type { MetricOutput } from \"../../core/types.js\";\n\n/**\n * Normalizes a score to 0-1 range\n */\nexport function normalizeScore(score: number, min = 0, max = 1): number {\n  const range = max - min;\n  if (range === 0) return 0;\n  return Math.max(0, Math.min(1, (score - min) / range));\n}\n\n/**\n * Converts a numeric score to a label based on thresholds\n */\nexport function scoreToLabel(\n  score: number,\n  thresholds: { label: string; min: number }[]\n): string {\n  // Sort thresholds by min descending\n  const sorted = [...thresholds].sort((a, b) => b.min - a.min);\n\n  for (const { label, min } of sorted) {\n    if (score >= min) {\n      return label;\n    }\n  }\n\n  return thresholds[thresholds.length - 1]?.label ?? \"unknown\";\n}\n\n/**\n * Creates a metric output from a score\n */\nexport function createMetricOutput(\n  id: string,\n  metric: string,\n  score: number,\n  labelThresholds?: { label: string; min: number }[]\n): MetricOutput {\n  const normalizedScore = normalizeScore(score);\n  const label = labelThresholds\n    ? scoreToLabel(normalizedScore, labelThresholds)\n    : normalizedScore >= 0.5\n      ? \"high\"\n      : \"low\";\n\n  return {\n    id,\n    metric,\n    score: normalizedScore,\n    label,\n  };\n}\n\n/**\n * Default thresholds for binary metrics\n */\nexport const BINARY_THRESHOLDS = [\n  { label: \"true\", min: 0.5 },\n  { label: \"false\", min: 0 },\n];\n\n/**\n * Default thresholds for severity metrics\n */\nexport const SEVERITY_THRESHOLDS = [\n  { label: \"high\", min: 0.7 },\n  { label: \"medium\", min: 0.4 },\n  { label: \"low\", min: 0 },\n];\n\n/**\n * Batches items for parallel processing\n */\nexport function batch<T>(items: T[], size: number): T[][] {\n  const batches: T[][] = [];\n  for (let i = 0; i < items.length; i += size) {\n    batches.push(items.slice(i, i + size));\n  }\n  return batches;\n}\n\n/**\n * Delays execution\n */\nexport function delay(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n","/**\n * Mock LLM client for testing\n *\n * Provides a configurable mock implementation of LLMClient for unit tests.\n */\n\nimport type { LLMClient, JSONSchema } from \"../../../core/types.js\";\n\n/**\n * Configuration for mock LLM client\n */\nexport interface MockLLMConfig {\n  /** Fixed response to return (can be string or object for JSON mode) */\n  response?: string | Record<string, unknown>;\n\n  /** Multiple responses for sequential calls */\n  responses?: Array<string | Record<string, unknown>>;\n\n  /** Delay in milliseconds before responding */\n  delay?: number;\n\n  /** Whether to throw an error */\n  shouldError?: boolean;\n\n  /** Error message to throw */\n  errorMessage?: string;\n\n  /** Function to validate prompts */\n  onPrompt?: (prompt: string) => void;\n}\n\n/**\n * Creates a mock LLM client for testing\n *\n * @example\n * ```ts\n * const mock = createMockLLMClient({\n *   response: JSON.stringify({ score: 0.8, reasoning: \"test\" }),\n *   delay: 100\n * });\n *\n * setLLMClient(mock);\n * ```\n */\nexport function createMockLLMClient(config: MockLLMConfig = {}): LLMClient {\n  const {\n    response,\n    responses,\n    delay = 0,\n    shouldError = false,\n    errorMessage = \"Mock LLM error\",\n    onPrompt,\n  } = config;\n\n  let callCount = 0;\n\n  const getResponse = (): string | Record<string, unknown> => {\n    if (responses && responses.length > 0) {\n      const resp = responses[Math.min(callCount, responses.length - 1)];\n      if (!resp) {\n        return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n      }\n      callCount++;\n      return resp;\n    }\n\n    if (response !== undefined) {\n      return response;\n    }\n\n    // Default response\n    return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n  };\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response\n      const resp = getResponse();\n      return typeof resp === \"string\" ? resp : JSON.stringify(resp);\n    },\n\n    async completeStructured<T>(prompt: string, _schema: JSONSchema): Promise<T> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response as object\n      const resp = getResponse();\n      if (typeof resp === \"string\") {\n        try {\n          return JSON.parse(resp) as T;\n        } catch {\n          throw new Error(`Mock response is not valid JSON: ${resp}`);\n        }\n      }\n\n      return resp as T;\n    },\n  };\n}\n\n/**\n * Creates a mock client that returns sequential responses\n *\n * Useful for testing multiple calls with different responses.\n *\n * @example\n * ```ts\n * const mock = createSequentialMockClient([\n *   { score: 0.2, reasoning: \"First call\" },\n *   { score: 0.8, reasoning: \"Second call\" }\n * ]);\n * ```\n */\nexport function createSequentialMockClient(\n  responses: Array<string | Record<string, unknown>>,\n  options: { delay?: number } = {}\n): LLMClient {\n  return createMockLLMClient({\n    responses,\n    delay: options.delay,\n  });\n}\n\n/**\n * Creates a mock client that always errors\n *\n * Useful for testing error handling.\n */\nexport function createErrorMockClient(errorMessage = \"Mock LLM error\"): LLMClient {\n  return createMockLLMClient({\n    shouldError: true,\n    errorMessage,\n  });\n}\n\n/**\n * Creates a spy mock client that records all prompts\n *\n * Useful for testing what prompts are being sent to the LLM.\n *\n * @example\n * ```ts\n * const { client, prompts } = createSpyMockClient({ score: 0.5 });\n * await metric({ outputs, context, llmClient: client });\n * console.log(prompts); // See all prompts that were sent\n * ```\n */\nexport function createSpyMockClient(\n  response: string | Record<string, unknown>\n): { client: LLMClient; prompts: string[] } {\n  const prompts: string[] = [];\n\n  const client = createMockLLMClient({\n    response,\n    onPrompt: (prompt) => prompts.push(prompt),\n  });\n\n  return { client, prompts };\n}\n"]}

package/dist/metrics/opinionated/index.cjs

@@ -0,0 +1,24 @@
+'use strict';
+
+var chunkY23VHTD3_cjs = require('../../chunk-Y23VHTD3.cjs');
+
+
+
+Object.defineProperty(exports, "faithfulness", {
+  enumerable: true,
+  get: function () { return chunkY23VHTD3_cjs.faithfulness; }
+});
+Object.defineProperty(exports, "hallucination", {
+  enumerable: true,
+  get: function () { return chunkY23VHTD3_cjs.hallucination; }
+});
+Object.defineProperty(exports, "relevance", {
+  enumerable: true,
+  get: function () { return chunkY23VHTD3_cjs.relevance; }
+});
+Object.defineProperty(exports, "toxicity", {
+  enumerable: true,
+  get: function () { return chunkY23VHTD3_cjs.toxicity; }
+});
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/metrics/opinionated/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"index.cjs"}

package/dist/metrics/opinionated/index.d.cts

@@ -0,0 +1,163 @@
+import { a as MetricConfig, b as MetricOutput } from '../../types-C71p0wzM.cjs';
+
+/**
+ * Hallucination detection metric (LLM-based)
+ *
+ * Detects statements in the output that are not supported by the provided context.
+ * Uses LLM evaluation for accurate hallucination detection.
+ */
+
+/**
+ * Configuration for hallucination metric
+ */
+interface HallucinationConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Context/source material that outputs should be faithful to */
+    context: string[];
+}
+/**
+ * Detects potential hallucinations by checking if output content
+ * is supported by the provided context.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, hallucination } from "evalsense/metrics";
+ *
+ * // Configure LLM client once
+ * setLLMClient({
+ *   async complete(prompt) {
+ *     return await yourLLM.generate(prompt);
+ *   }
+ * });
+ *
+ * // Use the metric
+ * const results = await hallucination({
+ *   outputs: [{ id: "1", output: "The capital of France is Paris." }],
+ *   context: ["France is a country in Europe. Its capital is Paris."]
+ * });
+ * ```
+ */
+declare function hallucination(config: HallucinationConfig): Promise<MetricOutput[]>;
+
+/**
+ * Relevance metric (LLM-based)
+ *
+ * Measures how relevant the output is to the input query.
+ * Uses LLM evaluation for accurate relevance assessment.
+ */
+
+/**
+ * Configuration for relevance metric
+ */
+interface RelevanceConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Queries that the outputs should be relevant to */
+    query: string[];
+}
+/**
+ * Measures the relevance of outputs to their queries.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, relevance } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await relevance({
+ *   outputs: [{ id: "1", output: "Paris is the capital of France." }],
+ *   query: ["What is the capital of France?"]
+ * });
+ * ```
+ */
+declare function relevance(config: RelevanceConfig): Promise<MetricOutput[]>;
+
+/**
+ * Faithfulness metric (LLM-based)
+ *
+ * Measures how faithful the output is to the source material.
+ * Uses LLM evaluation to detect contradictions and misrepresentations.
+ */
+
+/**
+ * Configuration for faithfulness metric
+ */
+interface FaithfulnessConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Source material that outputs should be faithful to */
+    source: string[];
+}
+/**
+ * Measures the faithfulness of outputs to their source material.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, faithfulness } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await faithfulness({
+ *   outputs: [{ id: "1", output: "The document discusses climate change." }],
+ *   source: ["This report covers the impacts of climate change on biodiversity."]
+ * });
+ * ```
+ */
+declare function faithfulness(config: FaithfulnessConfig): Promise<MetricOutput[]>;
+
+/**
+ * Toxicity detection metric (LLM-based)
+ *
+ * Detects potentially toxic, harmful, or inappropriate content.
+ * Uses LLM evaluation for nuanced toxicity detection.
+ */
+
+/**
+ * Configuration for toxicity metric
+ */
+interface ToxicityConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+}
+/**
+ * Detects potential toxicity in outputs.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, toxicity } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await toxicity({
+ *   outputs: [{ id: "1", output: "This is a friendly message." }]
+ * });
+ * ```
+ */
+declare function toxicity(config: ToxicityConfig): Promise<MetricOutput[]>;
+
+export { type FaithfulnessConfig, type HallucinationConfig, type RelevanceConfig, type ToxicityConfig, faithfulness, hallucination, relevance, toxicity };

package/dist/metrics/opinionated/index.d.ts

@@ -0,0 +1,163 @@
+import { a as MetricConfig, b as MetricOutput } from '../../types-C71p0wzM.js';
+
+/**
+ * Hallucination detection metric (LLM-based)
+ *
+ * Detects statements in the output that are not supported by the provided context.
+ * Uses LLM evaluation for accurate hallucination detection.
+ */
+
+/**
+ * Configuration for hallucination metric
+ */
+interface HallucinationConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Context/source material that outputs should be faithful to */
+    context: string[];
+}
+/**
+ * Detects potential hallucinations by checking if output content
+ * is supported by the provided context.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, hallucination } from "evalsense/metrics";
+ *
+ * // Configure LLM client once
+ * setLLMClient({
+ *   async complete(prompt) {
+ *     return await yourLLM.generate(prompt);
+ *   }
+ * });
+ *
+ * // Use the metric
+ * const results = await hallucination({
+ *   outputs: [{ id: "1", output: "The capital of France is Paris." }],
+ *   context: ["France is a country in Europe. Its capital is Paris."]
+ * });
+ * ```
+ */
+declare function hallucination(config: HallucinationConfig): Promise<MetricOutput[]>;
+
+/**
+ * Relevance metric (LLM-based)
+ *
+ * Measures how relevant the output is to the input query.
+ * Uses LLM evaluation for accurate relevance assessment.
+ */
+
+/**
+ * Configuration for relevance metric
+ */
+interface RelevanceConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Queries that the outputs should be relevant to */
+    query: string[];
+}
+/**
+ * Measures the relevance of outputs to their queries.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, relevance } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await relevance({
+ *   outputs: [{ id: "1", output: "Paris is the capital of France." }],
+ *   query: ["What is the capital of France?"]
+ * });
+ * ```
+ */
+declare function relevance(config: RelevanceConfig): Promise<MetricOutput[]>;
+
+/**
+ * Faithfulness metric (LLM-based)
+ *
+ * Measures how faithful the output is to the source material.
+ * Uses LLM evaluation to detect contradictions and misrepresentations.
+ */
+
+/**
+ * Configuration for faithfulness metric
+ */
+interface FaithfulnessConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+    /** Source material that outputs should be faithful to */
+    source: string[];
+}
+/**
+ * Measures the faithfulness of outputs to their source material.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, faithfulness } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await faithfulness({
+ *   outputs: [{ id: "1", output: "The document discusses climate change." }],
+ *   source: ["This report covers the impacts of climate change on biodiversity."]
+ * });
+ * ```
+ */
+declare function faithfulness(config: FaithfulnessConfig): Promise<MetricOutput[]>;
+
+/**
+ * Toxicity detection metric (LLM-based)
+ *
+ * Detects potentially toxic, harmful, or inappropriate content.
+ * Uses LLM evaluation for nuanced toxicity detection.
+ */
+
+/**
+ * Configuration for toxicity metric
+ */
+interface ToxicityConfig extends MetricConfig {
+    /** Model outputs to evaluate */
+    outputs: Array<{
+        id: string;
+        output: string;
+    }>;
+}
+/**
+ * Detects potential toxicity in outputs.
+ *
+ * This metric requires an LLM client. Set one globally with setLLMClient()
+ * or pass llmClient in the config.
+ *
+ * @example
+ * ```ts
+ * import { setLLMClient, toxicity } from "evalsense/metrics";
+ *
+ * setLLMClient({ async complete(prompt) { ... } });
+ *
+ * const results = await toxicity({
+ *   outputs: [{ id: "1", output: "This is a friendly message." }]
+ * });
+ * ```
+ */
+declare function toxicity(config: ToxicityConfig): Promise<MetricOutput[]>;
+
+export { type FaithfulnessConfig, type HallucinationConfig, type RelevanceConfig, type ToxicityConfig, faithfulness, hallucination, relevance, toxicity };

package/dist/metrics/opinionated/index.js

@@ -0,0 +1,3 @@
+export { faithfulness, hallucination, relevance, toxicity } from '../../chunk-BRPM6AB6.js';
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/metrics/opinionated/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"names":[],"mappings":"","file":"index.js"}