evalsense 0.3.2 → 0.4.0

package/dist/metrics/index.js

@@ -1,34 +1,8 @@
-import { parseJSONResponse } from '../chunk-Z3U6AUWX.js';
-export { batchItems, createJSONSchema, createLLMError, extractScore, faithfulness, fillPrompt, getLLMClient, hallucination, parseJSONResponse, relevance, requireLLMClient, resetLLMClient, setLLMClient, toxicity, validateResponse, withTimeout } from '../chunk-Z3U6AUWX.js';
-import { __require } from '../chunk-DGUM43GV.js';
+import { withLLMClient, resetLLMClient, resetDefaults } from '../chunk-TDGWDK2L.js';
+export { BINARY_THRESHOLDS, SEVERITY_THRESHOLDS, batch, batchItems, configureLLM, createAnthropicAdapter, createJSONSchema, createLLMError, createLLMMetric, createMetricOutput, createOpenAIAdapter, createOpenRouterAdapter, delay, extractScore, faithfulness, fillPrompt, getDefaults, getLLMClient, hallucination, normalizeScore, parseJSONResponse, relevance, requireLLMClient, resetDefaults, resetLLMClient, scoreToLabel, setDefaults, setLLMClient, toxicity, validateResponse, withLLMClient, withTimeout } from '../chunk-TDGWDK2L.js';
+import '../chunk-DGUM43GV.js';
 
-// src/metrics/custom/index.ts
-var customMetrics = /* @__PURE__ */ new Map();
-function registerMetric(name, fn) {
-  if (customMetrics.has(name)) {
-    throw new Error(`Metric "${name}" is already registered`);
-  }
-  customMetrics.set(name, fn);
-}
-function getMetric(name) {
-  return customMetrics.get(name);
-}
-async function runMetric(name, config) {
-  const fn = customMetrics.get(name);
-  if (!fn) {
-    throw new Error(`Metric "${name}" is not registered`);
-  }
-  return fn(config);
-}
-function listMetrics() {
-  return Array.from(customMetrics.keys());
-}
-function unregisterMetric(name) {
-  return customMetrics.delete(name);
-}
-function clearMetrics() {
-  customMetrics.clear();
-}
+// src/metrics/custom.ts
 function createPatternMetric(name, patterns, options = {}) {
   const { matchScore = 1, noMatchScore = 0 } = options;
   return async (config) => {
@@ -61,52 +35,7 @@ function createKeywordMetric(name, keywords, options = {}) {
   };
 }
 
-// src/metrics/utils/index.ts
-function normalizeScore(score, min = 0, max = 1) {
-  const range = max - min;
-  if (range === 0) return 0;
-  return Math.max(0, Math.min(1, (score - min) / range));
-}
-function scoreToLabel(score, thresholds) {
-  const sorted = [...thresholds].sort((a, b) => b.min - a.min);
-  for (const { label, min } of sorted) {
-    if (score >= min) {
-      return label;
-    }
-  }
-  return thresholds[thresholds.length - 1]?.label ?? "unknown";
-}
-function createMetricOutput(id, metric, score, labelThresholds) {
-  const normalizedScore = normalizeScore(score);
-  const label = labelThresholds ? scoreToLabel(normalizedScore, labelThresholds) : normalizedScore >= 0.5 ? "high" : "low";
-  return {
-    id,
-    metric,
-    score: normalizedScore,
-    label
-  };
-}
-var BINARY_THRESHOLDS = [
-  { label: "true", min: 0.5 },
-  { label: "false", min: 0 }
-];
-var SEVERITY_THRESHOLDS = [
-  { label: "high", min: 0.7 },
-  { label: "medium", min: 0.4 },
-  { label: "low", min: 0 }
-];
-function batch(items, size) {
-  const batches = [];
-  for (let i = 0; i < items.length; i += size) {
-    batches.push(items.slice(i, i + size));
-  }
-  return batches;
-}
-function delay(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-// src/metrics/llm/adapters/mock.ts
+// src/metrics/adapters/mock.ts
 function createMockLLMClient(config = {}) {
   const {
     response,
@@ -188,242 +117,25 @@ function createSpyMockClient(response) {
   return { client, prompts };
 }
 
-// src/metrics/llm/adapters/openai.ts
-function createOpenAIAdapter(apiKey, options = {}) {
-  const {
-    model = "gpt-4-turbo-preview",
-    temperature = 0,
-    maxTokens = 4096,
-    baseURL,
-    organization,
-    timeout = 3e4
-  } = options;
-  if (!apiKey) {
-    throw new Error(
-      "OpenAI API key is required. Get one at https://platform.openai.com/api-keys"
-    );
-  }
-  let OpenAI;
-  let openaiClient;
-  function ensureClient() {
-    if (openaiClient) return openaiClient;
-    try {
-      OpenAI = __require("openai").default || __require("openai");
-    } catch (error) {
-      throw new Error(
-        "OpenAI SDK not found. Install it with: npm install openai\nVisit https://github.com/openai/openai-node for documentation."
-      );
-    }
-    openaiClient = new OpenAI({
-      apiKey,
-      baseURL,
-      organization,
-      timeout
-    });
-    return openaiClient;
-  }
-  return {
-    async complete(prompt) {
-      const client = ensureClient();
-      try {
-        const response = await client.chat.completions.create({
-          model,
-          messages: [{ role: "user", content: prompt }],
-          temperature,
-          max_tokens: maxTokens
-        });
-        return response.choices[0]?.message?.content ?? "";
-      } catch (error) {
-        const errorMessage = error?.message || error?.error?.message || String(error);
-        throw new Error(
-          `OpenAI API error (model: ${model}): ${errorMessage}
-Check your API key and quota at https://platform.openai.com/account/usage`
-        );
-      }
-    },
-    async completeStructured(prompt, _schema) {
-      const client = ensureClient();
-      try {
-        const response = await client.chat.completions.create({
-          model,
-          messages: [{ role: "user", content: prompt }],
-          response_format: { type: "json_object" },
-          temperature,
-          max_tokens: maxTokens
-        });
-        const text = response.choices[0]?.message?.content ?? "{}";
-        return JSON.parse(text);
-      } catch (error) {
-        const errorMessage = error?.message || error?.error?.message || String(error);
-        throw new Error(
-          `OpenAI API error (model: ${model}): ${errorMessage}
-Check your API key and quota at https://platform.openai.com/account/usage`
-        );
-      }
-    }
-  };
-}
-
-// src/metrics/llm/adapters/anthropic.ts
-function createAnthropicAdapter(apiKey, options = {}) {
-  const {
-    model = "claude-3-5-sonnet-20241022",
-    maxTokens = 4096,
-    temperature = 0,
-    timeout = 3e4
-  } = options;
-  if (!apiKey) {
-    throw new Error(
-      "Anthropic API key is required. Get one at https://console.anthropic.com/"
-    );
-  }
-  if (temperature < 0 || temperature > 1) {
-    throw new Error(`Anthropic temperature must be between 0 and 1, got ${temperature}`);
-  }
-  let Anthropic;
-  let anthropicClient;
-  function ensureClient() {
-    if (anthropicClient) return anthropicClient;
-    try {
-      Anthropic = __require("@anthropic-ai/sdk").default || __require("@anthropic-ai/sdk");
-    } catch (error) {
-      throw new Error(
-        "Anthropic SDK not found. Install it with: npm install @anthropic-ai/sdk\nVisit https://github.com/anthropics/anthropic-sdk-typescript for documentation."
-      );
-    }
-    anthropicClient = new Anthropic({
-      apiKey,
-      timeout
-    });
-    return anthropicClient;
-  }
-  return {
-    async complete(prompt) {
-      const client = ensureClient();
-      try {
-        const message = await client.messages.create({
-          model,
-          max_tokens: maxTokens,
-          temperature,
-          messages: [{ role: "user", content: prompt }]
-        });
-        const firstBlock = message.content[0];
-        return firstBlock?.type === "text" ? firstBlock.text : "";
-      } catch (error) {
-        const errorMessage = error?.message || error?.error?.message || String(error);
-        throw new Error(
-          `Anthropic API error (model: ${model}): ${errorMessage}
-Check your API key and usage at https://console.anthropic.com/`
-        );
-      }
-    },
-    async completeStructured(prompt, schema) {
-      const jsonPrompt = prompt + `
-
-IMPORTANT: Respond with valid JSON only. No markdown, no explanation. The JSON must match this schema: ${JSON.stringify(schema)}`;
-      const response = await this.complete(jsonPrompt);
-      try {
-        return parseJSONResponse(response);
-      } catch (error) {
-        throw new Error(
-          `Failed to parse Anthropic response as JSON: ${error.message}
-Response preview: ${response.substring(0, 200)}...`
-        );
-      }
-    }
-  };
-}
-
-// src/metrics/llm/adapters/openrouter.ts
-function createOpenRouterAdapter(apiKey, options = {}) {
-  const {
-    model = "anthropic/claude-3.5-sonnet",
-    temperature = 0,
-    maxTokens = 4096,
-    appName = "evalsense",
-    siteUrl,
-    timeout = 3e4
-  } = options;
-  if (!apiKey) {
-    throw new Error("OpenRouter API key is required. Get one at https://openrouter.ai/keys");
-  }
-  const baseURL = "https://openrouter.ai/api/v1";
-  async function callAPI(messages, jsonMode = false) {
-    const headers = {
-      Authorization: `Bearer ${apiKey}`,
-      "Content-Type": "application/json",
-      "HTTP-Referer": siteUrl || "https://github.com/evalsense/evalsense",
-      "X-Title": appName
-    };
-    const body = {
-      model,
-      messages,
-      temperature,
-      max_tokens: maxTokens
-    };
-    if (jsonMode) {
-      body.response_format = { type: "json_object" };
-    }
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), timeout);
-    try {
-      const response = await fetch(`${baseURL}/chat/completions`, {
-        method: "POST",
-        headers,
-        body: JSON.stringify(body),
-        signal: controller.signal
-      });
-      clearTimeout(timeoutId);
-      if (!response.ok) {
-        const errorData = await response.json().catch(() => ({}));
-        const errorMessage = errorData.error?.message || response.statusText || "Unknown error";
-        throw new Error(`OpenRouter API error (${response.status}): ${errorMessage}`);
-      }
-      const data = await response.json();
-      return data.choices?.[0]?.message?.content ?? "";
-    } catch (error) {
-      clearTimeout(timeoutId);
-      if (error.name === "AbortError") {
-        throw new Error(`OpenRouter request timed out after ${timeout}ms (model: ${model})`);
-      }
-      const errorMessage = error?.message || String(error);
-      throw new Error(
-        `OpenRouter API error (model: ${model}): ${errorMessage}
-Check your API key and credits at https://openrouter.ai/activity`
-      );
-    }
-  }
-  return {
-    async complete(prompt) {
-      return callAPI([{ role: "user", content: prompt }], false);
-    },
-    async completeStructured(prompt, schema) {
-      let response;
-      try {
-        response = await callAPI(
-          [{ role: "user", content: prompt }],
-          true
-          // Enable JSON mode
-        );
-      } catch (error) {
-        const jsonPrompt = prompt + `
-
-IMPORTANT: Respond with valid JSON only. No markdown, no explanation. The JSON must match this schema: ${JSON.stringify(schema)}`;
-        response = await callAPI([{ role: "user", content: jsonPrompt }], false);
-      }
-      try {
-        return parseJSONResponse(response);
-      } catch (error) {
-        throw new Error(
-          `Failed to parse OpenRouter response as JSON: ${error.message}
-Model: ${model}
-Response preview: ${response.substring(0, 200)}...`
-        );
-      }
-    }
-  };
-}
+// src/metrics/index.ts
+var testing = {
+  /** Resets global LLM client and defaults */
+  reset: () => {
+    resetLLMClient();
+    resetDefaults();
+  },
+  /** Creates a mock LLM client */
+  mock: createMockLLMClient,
+  /** Executes function with scoped LLM client */
+  withClient: withLLMClient,
+  /** Creates a mock client that returns sequential responses */
+  sequentialMock: createSequentialMockClient,
+  /** Creates a mock client that always errors */
+  errorMock: createErrorMockClient,
+  /** Creates a spy mock client that records all prompts */
+  spyMock: createSpyMockClient
+};
 
-export { BINARY_THRESHOLDS, SEVERITY_THRESHOLDS, batch, clearMetrics, createAnthropicAdapter, createErrorMockClient, createKeywordMetric, createMetricOutput, createMockLLMClient, createOpenAIAdapter, createOpenRouterAdapter, createPatternMetric, createSequentialMockClient, createSpyMockClient, delay, getMetric, listMetrics, normalizeScore, registerMetric, runMetric, scoreToLabel, unregisterMetric };
+export { createErrorMockClient, createKeywordMetric, createMockLLMClient, createPatternMetric, createSequentialMockClient, createSpyMockClient, testing };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/metrics/custom/index.ts","../../src/metrics/utils/index.ts","../../src/metrics/llm/adapters/mock.ts","../../src/metrics/llm/adapters/openai.ts","../../src/metrics/llm/adapters/anthropic.ts","../../src/metrics/llm/adapters/openrouter.ts"],"names":["delay"],"mappings":";;;;;AASA,IAAM,aAAA,uBAAoB,GAAA,EAAsB;AAiBzC,SAAS,cAAA,CAAe,MAAc,EAAA,EAAoB;AAC/D,EAAA,IAAI,aAAA,CAAc,GAAA,CAAI,IAAI,CAAA,EAAG;AAC3B,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,QAAA,EAAW,IAAI,CAAA,uBAAA,CAAyB,CAAA;AAAA,EAC1D;AACA,EAAA,aAAA,CAAc,GAAA,CAAI,MAAM,EAAE,CAAA;AAC5B;AAKO,SAAS,UAAU,IAAA,EAAoC;AAC5D,EAAA,OAAO,aAAA,CAAc,IAAI,IAAI,CAAA;AAC/B;AAKA,eAAsB,SAAA,CAAU,MAAc,MAAA,EAA+C;AAC3F,EAAA,MAAM,EAAA,GAAK,aAAA,CAAc,GAAA,CAAI,IAAI,CAAA;AACjC,EAAA,IAAI,CAAC,EAAA,EAAI;AACP,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,QAAA,EAAW,IAAI,CAAA,mBAAA,CAAqB,CAAA;AAAA,EACtD;AACA,EAAA,OAAO,GAAG,MAAM,CAAA;AAClB;AAKO,SAAS,WAAA,GAAwB;AACtC,EAAA,OAAO,KAAA,CAAM,IAAA,CAAK,aAAA,CAAc,IAAA,EAAM,CAAA;AACxC;AAKO,SAAS,iBAAiB,IAAA,EAAuB;AACtD,EAAA,OAAO,aAAA,CAAc,OAAO,IAAI,CAAA;AAClC;AAKO,SAAS,YAAA,GAAqB;AACnC,EAAA,aAAA,CAAc,KAAA,EAAM;AACtB;AAKO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA0D,EAAC,EACjD;AACV,EAAA,MAAM,EAAE,UAAA,GAAa,CAAA,EAAG,YAAA,GAAe,GAAE,GAAI,OAAA;AAE7C,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,QAAA,GAAW,SAAS,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,IAAA,CAAK,CAAA,CAAE,MAAM,CAAC,CAAA;AACtD,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA,EAAO,WAAW,UAAA,GAAa,YAAA;AAAA,QAC/B,KAAA,EAAO,WAAW,UAAA,GAAa;AAAA,OACjC;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;AAKO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA2D,EAAC,EAClD;AACV,EAAA,MAAM,EAAE,aAAA,GAAgB,KAAA,EAAO,SAAA,GAAY,KAAI,GAAI,OAAA;AAEnD,EAAA,MAAM,kBAAA,GAAqB,gBAAgB,QAAA,GAAW,QAAA,CAAS,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,WAAA,EAAa,CAAA;AAEzF,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,OAAO,aAAA,GAAgB,CAAA,CAAE,MAAA,GAAS,CAAA,CAAE,OAAO,WAAA,EAAY;AAC7D,MAAA,MAAM,OAAA,GAAU,mBAAmB,MAAA,CAAO,CAAC,MAAM,IAAA,CAAK,QAAA,CAAS,CAAC,CAAC,CAAA;AACjE,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,GAAS,QAAA,CAAS,MAAA;AAExC,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA;AAAA,QACA,KAAA,EAAO,KAAA,IAAS,SAAA,GAAY,UAAA,GAAa;AAAA,OAC3C;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;;;AChHO,SAAS,cAAA,CAAe,KAAA,EAAe,GAAA,GAAM,CAAA,EAAG,MAAM,CAAA,EAAW;AACtE,EAAA,MAAM,QAAQ,GAAA,GAAM,GAAA;AACpB,EAAA,IAAI,KAAA,KAAU,GAAG,OAAO,CAAA;AACxB,EAAA,OAAO,IAAA,CAAK,IAAI,CAAA,EAAG,IAAA,CAAK,IAAI,CAAA,EAAA,CAAI,KAAA,GAAQ,GAAA,IAAO,KAAK,CAAC,CAAA;AACvD;AAKO,SAAS,YAAA,CAAa,OAAe,UAAA,EAAsD;AAEhG,EAAA,MAAM,MAAA,GAAS,CAAC,GAAG,UAAU,CAAA,CAAE,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,GAAA,GAAM,CAAA,CAAE,GAAG,CAAA;AAE3D,EAAA,KAAA,MAAW,EAAE,KAAA,EAAO,GAAA,EAAI,IAAK,MAAA,EAAQ;AACnC,IAAA,IAAI,SAAS,GAAA,EAAK;AAChB,MAAA,OAAO,KAAA;AAAA,IACT;AAAA,EACF;AAEA,EAAA,OAAO,UAAA,CAAW,UAAA,CAAW,MAAA,GAAS,CAAC,GAAG,KAAA,IAAS,SAAA;AACrD;AAKO,SAAS,kBAAA,CACd,EAAA,EACA,MAAA,EACA,KAAA,EACA,eAAA,EACc;AACd,EAAA,MAAM,eAAA,GAAkB,eAAe,KAAK,CAAA;AAC5C,EAAA,MAAM,KAAA,GAAQ,kBACV,YAAA,CAAa,eAAA,EAAiB,eAAe,CAAA,GAC7C,eAAA,IAAmB,MACjB,MAAA,GACA,KAAA;AAEN,EAAA,OAAO;AAAA,IACL,EAAA;AAAA,IACA,MAAA;AAAA,IACA,KAAA,EAAO,eAAA;AAAA,IACP;AAAA,GACF;AACF;AAKO,IAAM,iBAAA,GAAoB;AAAA,EAC/B,EAAE,KAAA,EAAO,MAAA,EAAQ,GAAA,EAAK,GAAA,EAAI;AAAA,EAC1B,EAAE,KAAA,EAAO,OAAA,EAAS,GAAA,EAAK,CAAA;AACzB;AAKO,IAAM,mBAAA,GAAsB;AAAA,EACjC,EAAE,KAAA,EAAO,MAAA,EAAQ,GAAA,EAAK,GAAA,EAAI;AAAA,EAC1B,EAAE,KAAA,EAAO,QAAA,EAAU,GAAA,EAAK,GAAA,EAAI;AAAA,EAC5B,EAAE,KAAA,EAAO,KAAA,EAAO,GAAA,EAAK,CAAA;AACvB;AAKO,SAAS,KAAA,CAAS,OAAY,IAAA,EAAqB;AACxD,EAAA,MAAM,UAAiB,EAAC;AACxB,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,KAAA,CAAM,MAAA,EAAQ,KAAK,IAAA,EAAM;AAC3C,IAAA,OAAA,CAAQ,KAAK,KAAA,CAAM,KAAA,CAAM,CAAA,EAAG,CAAA,GAAI,IAAI,CAAC,CAAA;AAAA,EACvC;AACA,EAAA,OAAO,OAAA;AACT;AAKO,SAAS,MAAM,EAAA,EAA2B;AAC/C,EAAA,OAAO,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAAS,EAAE,CAAC,CAAA;AACzD;;;AC5CO,SAAS,mBAAA,CAAoB,MAAA,GAAwB,EAAC,EAAc;AACzE,EAAA,MAAM;AAAA,IACJ,QAAA;AAAA,IACA,SAAA;AAAA,IACA,OAAAA,MAAAA,GAAQ,CAAA;AAAA,IACR,WAAA,GAAc,KAAA;AAAA,IACd,YAAA,GAAe,gBAAA;AAAA,IACf;AAAA,GACF,GAAI,MAAA;AAEJ,EAAA,IAAI,SAAA,GAAY,CAAA;AAEhB,EAAA,MAAM,cAAc,MAAwC;AAC1D,IAAA,IAAI,SAAA,IAAa,SAAA,CAAU,MAAA,GAAS,CAAA,EAAG;AACrC,MAAA,MAAM,IAAA,GAAO,UAAU,IAAA,CAAK,GAAA,CAAI,WAAW,SAAA,CAAU,MAAA,GAAS,CAAC,CAAC,CAAA;AAChE,MAAA,IAAI,CAAC,IAAA,EAAM;AACT,QAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,MAClE;AACA,MAAA,SAAA,EAAA;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAI,aAAa,MAAA,EAAW;AAC1B,MAAA,OAAO,QAAA;AAAA,IACT;AAGA,IAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,EAClE,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAE9C,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,OAAO,OAAO,IAAA,KAAS,QAAA,GAAW,IAAA,GAAO,IAAA,CAAK,UAAU,IAAI,CAAA;AAAA,IAC9D,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,OAAA,EAAiC;AAE3E,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,IAAI,OAAO,SAAS,QAAA,EAAU;AAC5B,QAAA,IAAI;AACF,UAAA,OAAO,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,QACxB,CAAA,CAAA,MAAQ;AACN,UAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iCAAA,EAAoC,IAAI,CAAA,CAAE,CAAA;AAAA,QAC5D;AAAA,MACF;AAEA,MAAA,OAAO,IAAA;AAAA,IACT;AAAA,GACF;AACF;AAeO,SAAS,0BAAA,CACd,SAAA,EACA,OAAA,GAA8B,EAAC,EACpB;AACX,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,SAAA;AAAA,IACA,OAAO,OAAA,CAAQ;AAAA,GAChB,CAAA;AACH;AAOO,SAAS,qBAAA,CAAsB,eAAe,gBAAA,EAA6B;AAChF,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,WAAA,EAAa,IAAA;AAAA,IACb;AAAA,GACD,CAAA;AACH;AAcO,SAAS,oBAAoB,QAAA,EAGlC;AACA,EAAA,MAAM,UAAoB,EAAC;AAE3B,EAAA,MAAM,SAAS,mBAAA,CAAoB;AAAA,IACjC,QAAA;AAAA,IACA,QAAA,EAAU,CAAC,MAAA,KAAW,OAAA,CAAQ,KAAK,MAAM;AAAA,GAC1C,CAAA;AAED,EAAA,OAAO,EAAE,QAAQ,OAAA,EAAQ;AAC3B;;;AC/FO,SAAS,mBAAA,CAAoB,MAAA,EAAgB,OAAA,GAAgC,EAAC,EAAc;AACjG,EAAA,MAAM;AAAA,IACJ,KAAA,GAAQ,qBAAA;AAAA,IACR,WAAA,GAAc,CAAA;AAAA,IACd,SAAA,GAAY,IAAA;AAAA,IACZ,OAAA;AAAA,IACA,YAAA;AAAA,IACA,OAAA,GAAU;AAAA,GACZ,GAAI,OAAA;AAGJ,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,KAAA;AAAA,MACR;AAAA,KACF;AAAA,EACF;AAGA,EAAA,IAAI,MAAA;AACJ,EAAA,IAAI,YAAA;AAEJ,EAAA,SAAS,YAAA,GAAe;AACtB,IAAA,IAAI,cAAc,OAAO,YAAA;AAEzB,IAAA,IAAI;AAEF,MAAA,MAAA,GAAS,SAAA,CAAQ,QAAQ,CAAA,CAAE,OAAA,IAAW,UAAQ,QAAQ,CAAA;AAAA,IACxD,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OAEF;AAAA,IACF;AAEA,IAAA,YAAA,GAAe,IAAI,MAAA,CAAO;AAAA,MACxB,MAAA;AAAA,MACA,OAAA;AAAA,MACA,YAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,OAAO,YAAA;AAAA,EACT;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAC9C,MAAA,MAAM,SAAS,YAAA,EAAa;AAE5B,MAAA,IAAI;AACF,QAAA,MAAM,QAAA,GAAW,MAAM,MAAA,CAAO,IAAA,CAAK,YAAY,MAAA,CAAO;AAAA,UACpD,KAAA;AAAA,UACA,UAAU,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,QAAQ,CAAA;AAAA,UAC5C,WAAA;AAAA,UACA,UAAA,EAAY;AAAA,SACb,CAAA;AAED,QAAA,OAAO,QAAA,CAAS,OAAA,CAAQ,CAAC,CAAA,EAAG,SAAS,OAAA,IAAW,EAAA;AAAA,MAClD,SAAS,KAAA,EAAY;AAEnB,QAAA,MAAM,eAAe,KAAA,EAAO,OAAA,IAAW,OAAO,KAAA,EAAO,OAAA,IAAW,OAAO,KAAK,CAAA;AAC5E,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yBAAA,EAA4B,KAAK,CAAA,GAAA,EAAM,YAAY;AAAA,yEAAA;AAAA,SAErD;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,OAAA,EAAiC;AAC3E,MAAA,MAAM,SAAS,YAAA,EAAa;AAE5B,MAAA,IAAI;AACF,QAAA,MAAM,QAAA,GAAW,MAAM,MAAA,CAAO,IAAA,CAAK,YAAY,MAAA,CAAO;AAAA,UACpD,KAAA;AAAA,UACA,UAAU,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,QAAQ,CAAA;AAAA,UAC5C,eAAA,EAAiB,EAAE,IAAA,EAAM,aAAA,EAAc;AAAA,UACvC,WAAA;AAAA,UACA,UAAA,EAAY;AAAA,SACb,CAAA;AAED,QAAA,MAAM,OAAO,QAAA,CAAS,OAAA,CAAQ,CAAC,CAAA,EAAG,SAAS,OAAA,IAAW,IAAA;AACtD,QAAA,OAAO,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,MACxB,SAAS,KAAA,EAAY;AACnB,QAAA,MAAM,eAAe,KAAA,EAAO,OAAA,IAAW,OAAO,KAAA,EAAO,OAAA,IAAW,OAAO,KAAK,CAAA;AAC5E,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,yBAAA,EAA4B,KAAK,CAAA,GAAA,EAAM,YAAY;AAAA,yEAAA;AAAA,SAErD;AAAA,MACF;AAAA,IACF;AAAA,GACF;AACF;;;ACzGO,SAAS,sBAAA,CACd,MAAA,EACA,OAAA,GAAmC,EAAC,EACzB;AACX,EAAA,MAAM;AAAA,IACJ,KAAA,GAAQ,4BAAA;AAAA,IACR,SAAA,GAAY,IAAA;AAAA,IACZ,WAAA,GAAc,CAAA;AAAA,IACd,OAAA,GAAU;AAAA,GACZ,GAAI,OAAA;AAGJ,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,KAAA;AAAA,MACR;AAAA,KACF;AAAA,EACF;AAGA,EAAA,IAAI,WAAA,GAAc,CAAA,IAAK,WAAA,GAAc,CAAA,EAAG;AACtC,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mDAAA,EAAsD,WAAW,CAAA,CAAE,CAAA;AAAA,EACrF;AAGA,EAAA,IAAI,SAAA;AACJ,EAAA,IAAI,eAAA;AAEJ,EAAA,SAAS,YAAA,GAAe;AACtB,IAAA,IAAI,iBAAiB,OAAO,eAAA;AAE5B,IAAA,IAAI;AAEF,MAAA,SAAA,GAAY,SAAA,CAAQ,mBAAmB,CAAA,CAAE,OAAA,IAAW,UAAQ,mBAAmB,CAAA;AAAA,IACjF,SAAS,KAAA,EAAO;AACd,MAAA,MAAM,IAAI,KAAA;AAAA,QACR;AAAA,OAEF;AAAA,IACF;AAEA,IAAA,eAAA,GAAkB,IAAI,SAAA,CAAU;AAAA,MAC9B,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAED,IAAA,OAAO,eAAA;AAAA,EACT;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAC9C,MAAA,MAAM,SAAS,YAAA,EAAa;AAE5B,MAAA,IAAI;AACF,QAAA,MAAM,OAAA,GAAU,MAAM,MAAA,CAAO,QAAA,CAAS,MAAA,CAAO;AAAA,UAC3C,KAAA;AAAA,UACA,UAAA,EAAY,SAAA;AAAA,UACZ,WAAA;AAAA,UACA,UAAU,CAAC,EAAE,MAAM,MAAA,EAAQ,OAAA,EAAS,QAAQ;AAAA,SAC7C,CAAA;AAGD,QAAA,MAAM,UAAA,GAAa,OAAA,CAAQ,OAAA,CAAQ,CAAC,CAAA;AACpC,QAAA,OAAO,UAAA,EAAY,IAAA,KAAS,MAAA,GAAS,UAAA,CAAW,IAAA,GAAO,EAAA;AAAA,MACzD,SAAS,KAAA,EAAY;AACnB,QAAA,MAAM,eAAe,KAAA,EAAO,OAAA,IAAW,OAAO,KAAA,EAAO,OAAA,IAAW,OAAO,KAAK,CAAA;AAC5E,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,4BAAA,EAA+B,KAAK,CAAA,GAAA,EAAM,YAAY;AAAA,8DAAA;AAAA,SAExD;AAAA,MACF;AAAA,IACF,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,MAAA,EAAgC;AAG1E,MAAA,MAAM,aACJ,MAAA,GACA;;AAAA,uGAAA,EACoC,IAAA,CAAK,SAAA,CAAU,MAAM,CAAC,CAAA,CAAA;AAE5D,MAAA,MAAM,QAAA,GAAW,MAAM,IAAA,CAAK,QAAA,CAAS,UAAU,CAAA;AAE/C,MAAA,IAAI;AACF,QAAA,OAAO,kBAAqB,QAAQ,CAAA;AAAA,MACtC,SAAS,KAAA,EAAY;AACnB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,4CAAA,EAA+C,MAAM,OAAO;AAAA,kBAAA,EACrC,QAAA,CAAS,SAAA,CAAU,CAAA,EAAG,GAAG,CAAC,CAAA,GAAA;AAAA,SACnD;AAAA,MACF;AAAA,IACF;AAAA,GACF;AACF;;;AClEO,SAAS,uBAAA,CACd,MAAA,EACA,OAAA,GAAoC,EAAC,EAC1B;AACX,EAAA,MAAM;AAAA,IACJ,KAAA,GAAQ,6BAAA;AAAA,IACR,WAAA,GAAc,CAAA;AAAA,IACd,SAAA,GAAY,IAAA;AAAA,IACZ,OAAA,GAAU,WAAA;AAAA,IACV,OAAA;AAAA,IACA,OAAA,GAAU;AAAA,GACZ,GAAI,OAAA;AAGJ,EAAA,IAAI,CAAC,MAAA,EAAQ;AACX,IAAA,MAAM,IAAI,MAAM,uEAA4E,CAAA;AAAA,EAC9F;AAEA,EAAA,MAAM,OAAA,GAAU,8BAAA;AAEhB,EAAA,eAAe,OAAA,CACb,QAAA,EACA,QAAA,GAAoB,KAAA,EACH;AACjB,IAAA,MAAM,OAAA,GAAkC;AAAA,MACtC,aAAA,EAAe,UAAU,MAAM,CAAA,CAAA;AAAA,MAC/B,cAAA,EAAgB,kBAAA;AAAA,MAChB,gBAAgB,OAAA,IAAW,wCAAA;AAAA,MAC3B,SAAA,EAAW;AAAA,KACb;AAEA,IAAA,MAAM,IAAA,GAAY;AAAA,MAChB,KAAA;AAAA,MACA,QAAA;AAAA,MACA,WAAA;AAAA,MACA,UAAA,EAAY;AAAA,KACd;AAGA,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,IAAA,CAAK,eAAA,GAAkB,EAAE,IAAA,EAAM,aAAA,EAAc;AAAA,IAC/C;AAEA,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,MAAM,YAAY,UAAA,CAAW,MAAM,UAAA,CAAW,KAAA,IAAS,OAAO,CAAA;AAE9D,IAAA,IAAI;AACF,MAAA,MAAM,QAAA,GAAW,MAAM,KAAA,CAAM,CAAA,EAAG,OAAO,CAAA,iBAAA,CAAA,EAAqB;AAAA,QAC1D,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA;AAAA,QACA,IAAA,EAAM,IAAA,CAAK,SAAA,CAAU,IAAI,CAAA;AAAA,QACzB,QAAQ,UAAA,CAAW;AAAA,OACpB,CAAA;AAED,MAAA,YAAA,CAAa,SAAS,CAAA;AAEtB,MAAA,IAAI,CAAC,SAAS,EAAA,EAAI;AAChB,QAAA,MAAM,SAAA,GAAiB,MAAM,QAAA,CAAS,IAAA,GAAO,KAAA,CAAM,OAAO,EAAC,CAAE,CAAA;AAC7D,QAAA,MAAM,YAAA,GAAe,SAAA,CAAU,KAAA,EAAO,OAAA,IAAW,SAAS,UAAA,IAAc,eAAA;AACxE,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,sBAAA,EAAyB,SAAS,MAAM,CAAA,GAAA,EAAM,YAAY,CAAA,CAAE,CAAA;AAAA,MAC9E;AAEA,MAAA,MAAM,IAAA,GAAY,MAAM,QAAA,CAAS,IAAA,EAAK;AACtC,MAAA,OAAO,IAAA,CAAK,OAAA,GAAU,CAAC,CAAA,EAAG,SAAS,OAAA,IAAW,EAAA;AAAA,IAChD,SAAS,KAAA,EAAY;AACnB,MAAA,YAAA,CAAa,SAAS,CAAA;AAEtB,MAAA,IAAI,KAAA,CAAM,SAAS,YAAA,EAAc;AAC/B,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,mCAAA,EAAsC,OAAO,CAAA,WAAA,EAAc,KAAK,CAAA,CAAA,CAAG,CAAA;AAAA,MACrF;AAEA,MAAA,MAAM,YAAA,GAAe,KAAA,EAAO,OAAA,IAAW,MAAA,CAAO,KAAK,CAAA;AACnD,MAAA,MAAM,IAAI,KAAA;AAAA,QACR,CAAA,6BAAA,EAAgC,KAAK,CAAA,GAAA,EAAM,YAAY;AAAA,gEAAA;AAAA,OAEzD;AAAA,IACF;AAAA,EACF;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAC9C,MAAA,OAAO,OAAA,CAAQ,CAAC,EAAE,IAAA,EAAM,QAAQ,OAAA,EAAS,MAAA,EAAQ,CAAA,EAAG,KAAK,CAAA;AAAA,IAC3D,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,MAAA,EAAgC;AAE1E,MAAA,IAAI,QAAA;AACJ,MAAA,IAAI;AACF,QAAA,QAAA,GAAW,MAAM,OAAA;AAAA,UACf,CAAC,EAAE,IAAA,EAAM,MAAA,EAAQ,OAAA,EAAS,QAAQ,CAAA;AAAA,UAClC;AAAA;AAAA,SACF;AAAA,MACF,SAAS,KAAA,EAAO;AAEd,QAAA,MAAM,aACJ,MAAA,GACA;;AAAA,uGAAA,EACoC,IAAA,CAAK,SAAA,CAAU,MAAM,CAAC,CAAA,CAAA;AAC5D,QAAA,QAAA,GAAW,MAAM,OAAA,CAAQ,CAAC,EAAE,IAAA,EAAM,QAAQ,OAAA,EAAS,UAAA,EAAY,CAAA,EAAG,KAAK,CAAA;AAAA,MACzE;AAEA,MAAA,IAAI;AACF,QAAA,OAAO,kBAAqB,QAAQ,CAAA;AAAA,MACtC,SAAS,KAAA,EAAY;AACnB,QAAA,MAAM,IAAI,KAAA;AAAA,UACR,CAAA,6CAAA,EAAgD,MAAM,OAAO;AAAA,OAAA,EACjD,KAAK;AAAA,kBAAA,EACM,QAAA,CAAS,SAAA,CAAU,CAAA,EAAG,GAAG,CAAC,CAAA,GAAA;AAAA,SACnD;AAAA,MACF;AAAA,IACF;AAAA,GACF;AACF","file":"index.js","sourcesContent":["/**\n * Custom metric registration\n */\n\nimport type { MetricFn, MetricOutput, MetricConfig } from \"../../core/types.js\";\n\n/**\n * Registry of custom metrics\n */\nconst customMetrics = new Map<string, MetricFn>();\n\n/**\n * Registers a custom metric\n *\n * @example\n * ```ts\n * registerMetric(\"custom-relevance\", async ({ outputs, query }) => {\n *   // Custom evaluation logic\n *   return outputs.map(o => ({\n *     id: o.id,\n *     metric: \"custom-relevance\",\n *     score: evaluateRelevance(o.output, query),\n *   }));\n * });\n * ```\n */\nexport function registerMetric(name: string, fn: MetricFn): void {\n  if (customMetrics.has(name)) {\n    throw new Error(`Metric \"${name}\" is already registered`);\n  }\n  customMetrics.set(name, fn);\n}\n\n/**\n * Gets a registered custom metric\n */\nexport function getMetric(name: string): MetricFn | undefined {\n  return customMetrics.get(name);\n}\n\n/**\n * Runs a registered metric\n */\nexport async function runMetric(name: string, config: MetricConfig): Promise<MetricOutput[]> {\n  const fn = customMetrics.get(name);\n  if (!fn) {\n    throw new Error(`Metric \"${name}\" is not registered`);\n  }\n  return fn(config);\n}\n\n/**\n * Lists all registered custom metrics\n */\nexport function listMetrics(): string[] {\n  return Array.from(customMetrics.keys());\n}\n\n/**\n * Unregisters a metric (mainly for testing)\n */\nexport function unregisterMetric(name: string): boolean {\n  return customMetrics.delete(name);\n}\n\n/**\n * Clears all registered metrics (mainly for testing)\n */\nexport function clearMetrics(): void {\n  customMetrics.clear();\n}\n\n/**\n * Creates a simple string-matching metric\n */\nexport function createPatternMetric(\n  name: string,\n  patterns: RegExp[],\n  options: { matchScore?: number; noMatchScore?: number } = {}\n): MetricFn {\n  const { matchScore = 1, noMatchScore = 0 } = options;\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const hasMatch = patterns.some((p) => p.test(o.output));\n      return {\n        id: o.id,\n        metric: name,\n        score: hasMatch ? matchScore : noMatchScore,\n        label: hasMatch ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n\n/**\n * Creates a keyword-based metric\n */\nexport function createKeywordMetric(\n  name: string,\n  keywords: string[],\n  options: { caseSensitive?: boolean; threshold?: number } = {}\n): MetricFn {\n  const { caseSensitive = false, threshold = 0.5 } = options;\n\n  const normalizedKeywords = caseSensitive ? keywords : keywords.map((k) => k.toLowerCase());\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const text = caseSensitive ? o.output : o.output.toLowerCase();\n      const matches = normalizedKeywords.filter((k) => text.includes(k));\n      const score = matches.length / keywords.length;\n\n      return {\n        id: o.id,\n        metric: name,\n        score,\n        label: score >= threshold ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n","/**\n * Metric utilities\n */\n\nimport type { MetricOutput } from \"../../core/types.js\";\n\n/**\n * Normalizes a score to 0-1 range\n */\nexport function normalizeScore(score: number, min = 0, max = 1): number {\n  const range = max - min;\n  if (range === 0) return 0;\n  return Math.max(0, Math.min(1, (score - min) / range));\n}\n\n/**\n * Converts a numeric score to a label based on thresholds\n */\nexport function scoreToLabel(score: number, thresholds: { label: string; min: number }[]): string {\n  // Sort thresholds by min descending\n  const sorted = [...thresholds].sort((a, b) => b.min - a.min);\n\n  for (const { label, min } of sorted) {\n    if (score >= min) {\n      return label;\n    }\n  }\n\n  return thresholds[thresholds.length - 1]?.label ?? \"unknown\";\n}\n\n/**\n * Creates a metric output from a score\n */\nexport function createMetricOutput(\n  id: string,\n  metric: string,\n  score: number,\n  labelThresholds?: { label: string; min: number }[]\n): MetricOutput {\n  const normalizedScore = normalizeScore(score);\n  const label = labelThresholds\n    ? scoreToLabel(normalizedScore, labelThresholds)\n    : normalizedScore >= 0.5\n      ? \"high\"\n      : \"low\";\n\n  return {\n    id,\n    metric,\n    score: normalizedScore,\n    label,\n  };\n}\n\n/**\n * Default thresholds for binary metrics\n */\nexport const BINARY_THRESHOLDS = [\n  { label: \"true\", min: 0.5 },\n  { label: \"false\", min: 0 },\n];\n\n/**\n * Default thresholds for severity metrics\n */\nexport const SEVERITY_THRESHOLDS = [\n  { label: \"high\", min: 0.7 },\n  { label: \"medium\", min: 0.4 },\n  { label: \"low\", min: 0 },\n];\n\n/**\n * Batches items for parallel processing\n */\nexport function batch<T>(items: T[], size: number): T[][] {\n  const batches: T[][] = [];\n  for (let i = 0; i < items.length; i += size) {\n    batches.push(items.slice(i, i + size));\n  }\n  return batches;\n}\n\n/**\n * Delays execution\n */\nexport function delay(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n","/**\n * Mock LLM client for testing\n *\n * Provides a configurable mock implementation of LLMClient for unit tests.\n */\n\nimport type { LLMClient, JSONSchema } from \"../../../core/types.js\";\n\n/**\n * Configuration for mock LLM client\n */\nexport interface MockLLMConfig {\n  /** Fixed response to return (can be string or object for JSON mode) */\n  response?: string | Record<string, unknown>;\n\n  /** Multiple responses for sequential calls */\n  responses?: Array<string | Record<string, unknown>>;\n\n  /** Delay in milliseconds before responding */\n  delay?: number;\n\n  /** Whether to throw an error */\n  shouldError?: boolean;\n\n  /** Error message to throw */\n  errorMessage?: string;\n\n  /** Function to validate prompts */\n  onPrompt?: (prompt: string) => void;\n}\n\n/**\n * Creates a mock LLM client for testing\n *\n * @example\n * ```ts\n * const mock = createMockLLMClient({\n *   response: JSON.stringify({ score: 0.8, reasoning: \"test\" }),\n *   delay: 100\n * });\n *\n * setLLMClient(mock);\n * ```\n */\nexport function createMockLLMClient(config: MockLLMConfig = {}): LLMClient {\n  const {\n    response,\n    responses,\n    delay = 0,\n    shouldError = false,\n    errorMessage = \"Mock LLM error\",\n    onPrompt,\n  } = config;\n\n  let callCount = 0;\n\n  const getResponse = (): string | Record<string, unknown> => {\n    if (responses && responses.length > 0) {\n      const resp = responses[Math.min(callCount, responses.length - 1)];\n      if (!resp) {\n        return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n      }\n      callCount++;\n      return resp;\n    }\n\n    if (response !== undefined) {\n      return response;\n    }\n\n    // Default response\n    return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n  };\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response\n      const resp = getResponse();\n      return typeof resp === \"string\" ? resp : JSON.stringify(resp);\n    },\n\n    async completeStructured<T>(prompt: string, _schema: JSONSchema): Promise<T> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response as object\n      const resp = getResponse();\n      if (typeof resp === \"string\") {\n        try {\n          return JSON.parse(resp) as T;\n        } catch {\n          throw new Error(`Mock response is not valid JSON: ${resp}`);\n        }\n      }\n\n      return resp as T;\n    },\n  };\n}\n\n/**\n * Creates a mock client that returns sequential responses\n *\n * Useful for testing multiple calls with different responses.\n *\n * @example\n * ```ts\n * const mock = createSequentialMockClient([\n *   { score: 0.2, reasoning: \"First call\" },\n *   { score: 0.8, reasoning: \"Second call\" }\n * ]);\n * ```\n */\nexport function createSequentialMockClient(\n  responses: Array<string | Record<string, unknown>>,\n  options: { delay?: number } = {}\n): LLMClient {\n  return createMockLLMClient({\n    responses,\n    delay: options.delay,\n  });\n}\n\n/**\n * Creates a mock client that always errors\n *\n * Useful for testing error handling.\n */\nexport function createErrorMockClient(errorMessage = \"Mock LLM error\"): LLMClient {\n  return createMockLLMClient({\n    shouldError: true,\n    errorMessage,\n  });\n}\n\n/**\n * Creates a spy mock client that records all prompts\n *\n * Useful for testing what prompts are being sent to the LLM.\n *\n * @example\n * ```ts\n * const { client, prompts } = createSpyMockClient({ score: 0.5 });\n * await metric({ outputs, context, llmClient: client });\n * console.log(prompts); // See all prompts that were sent\n * ```\n */\nexport function createSpyMockClient(response: string | Record<string, unknown>): {\n  client: LLMClient;\n  prompts: string[];\n} {\n  const prompts: string[] = [];\n\n  const client = createMockLLMClient({\n    response,\n    onPrompt: (prompt) => prompts.push(prompt),\n  });\n\n  return { client, prompts };\n}\n","/**\n * Built-in OpenAI adapter for evalsense\n *\n * Provides a simple way to use OpenAI models without writing adapter code.\n *\n * @example\n * ```javascript\n * import { setLLMClient, createOpenAIAdapter } from 'evalsense/metrics';\n *\n * setLLMClient(createOpenAIAdapter(process.env.OPENAI_API_KEY, {\n *   model: 'gpt-4-turbo-preview',\n *   temperature: 0\n * }));\n * ```\n */\n\nimport type { LLMClient, JSONSchema } from \"../../../core/types.js\";\n\nexport interface OpenAIAdapterOptions {\n  /**\n   * OpenAI model to use\n   * @default \"gpt-4-turbo-preview\"\n   */\n  model?: string;\n\n  /**\n   * Temperature for generation (0-2)\n   * @default 0\n   */\n  temperature?: number;\n\n  /**\n   * Maximum tokens per completion\n   * @default 4096\n   */\n  maxTokens?: number;\n\n  /**\n   * API base URL (for Azure OpenAI or proxies)\n   * @default undefined (uses default OpenAI endpoint)\n   */\n  baseURL?: string;\n\n  /**\n   * Organization ID (optional)\n   */\n  organization?: string;\n\n  /**\n   * Request timeout in milliseconds\n   * @default 30000\n   */\n  timeout?: number;\n}\n\n/**\n * Creates an LLM client adapter for OpenAI.\n *\n * **Setup:**\n * 1. Install OpenAI SDK: `npm install openai`\n * 2. Get API key from https://platform.openai.com/api-keys\n * 3. Set environment variable: `export OPENAI_API_KEY=\"sk-...\"`\n *\n * **Model Options:**\n * - `gpt-4-turbo-preview` - Most capable, expensive\n * - `gpt-4` - High quality, expensive\n * - `gpt-3.5-turbo` - Fast and cheap (20x cheaper than GPT-4)\n *\n * @param apiKey - Your OpenAI API key\n * @param options - Configuration options\n * @returns LLM client for use with evalsense metrics\n *\n * @example\n * ```javascript\n * // Basic usage\n * const client = createOpenAIAdapter(process.env.OPENAI_API_KEY);\n * setLLMClient(client);\n *\n * // With custom model\n * const client = createOpenAIAdapter(process.env.OPENAI_API_KEY, {\n *   model: 'gpt-3.5-turbo',  // Cheaper model\n *   temperature: 0.3\n * });\n *\n * // With Azure OpenAI\n * const client = createOpenAIAdapter(process.env.AZURE_OPENAI_KEY, {\n *   baseURL: 'https://your-resource.openai.azure.com',\n *   model: 'gpt-4'\n * });\n * ```\n */\nexport function createOpenAIAdapter(apiKey: string, options: OpenAIAdapterOptions = {}): LLMClient {\n  const {\n    model = \"gpt-4-turbo-preview\",\n    temperature = 0,\n    maxTokens = 4096,\n    baseURL,\n    organization,\n    timeout = 30000,\n  } = options;\n\n  // Validate API key\n  if (!apiKey) {\n    throw new Error(\n      \"OpenAI API key is required. \" + \"Get one at https://platform.openai.com/api-keys\"\n    );\n  }\n\n  // Lazy-load OpenAI SDK (peer dependency)\n  let OpenAI: any;\n  let openaiClient: any;\n\n  function ensureClient() {\n    if (openaiClient) return openaiClient;\n\n    try {\n      // Try ESM import first\n      OpenAI = require(\"openai\").default || require(\"openai\");\n    } catch (error) {\n      throw new Error(\n        \"OpenAI SDK not found. Install it with: npm install openai\\n\" +\n          \"Visit https://github.com/openai/openai-node for documentation.\"\n      );\n    }\n\n    openaiClient = new OpenAI({\n      apiKey,\n      baseURL,\n      organization,\n      timeout,\n    });\n\n    return openaiClient;\n  }\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      const client = ensureClient();\n\n      try {\n        const response = await client.chat.completions.create({\n          model,\n          messages: [{ role: \"user\", content: prompt }],\n          temperature,\n          max_tokens: maxTokens,\n        });\n\n        return response.choices[0]?.message?.content ?? \"\";\n      } catch (error: any) {\n        // Enhance error message with context\n        const errorMessage = error?.message || error?.error?.message || String(error);\n        throw new Error(\n          `OpenAI API error (model: ${model}): ${errorMessage}\\n` +\n            `Check your API key and quota at https://platform.openai.com/account/usage`\n        );\n      }\n    },\n\n    async completeStructured<T>(prompt: string, _schema: JSONSchema): Promise<T> {\n      const client = ensureClient();\n\n      try {\n        const response = await client.chat.completions.create({\n          model,\n          messages: [{ role: \"user\", content: prompt }],\n          response_format: { type: \"json_object\" },\n          temperature,\n          max_tokens: maxTokens,\n        });\n\n        const text = response.choices[0]?.message?.content ?? \"{}\";\n        return JSON.parse(text) as T;\n      } catch (error: any) {\n        const errorMessage = error?.message || error?.error?.message || String(error);\n        throw new Error(\n          `OpenAI API error (model: ${model}): ${errorMessage}\\n` +\n            `Check your API key and quota at https://platform.openai.com/account/usage`\n        );\n      }\n    },\n  };\n}\n","/**\n * Built-in Anthropic (Claude) adapter for evalsense\n *\n * Provides a simple way to use Claude models without writing adapter code.\n *\n * @example\n * ```javascript\n * import { setLLMClient, createAnthropicAdapter } from 'evalsense/metrics';\n *\n * setLLMClient(createAnthropicAdapter(process.env.ANTHROPIC_API_KEY, {\n *   model: 'claude-3-5-sonnet-20241022'\n * }));\n * ```\n */\n\nimport type { LLMClient, JSONSchema } from \"../../../core/types.js\";\nimport { parseJSONResponse } from \"../utils.js\";\n\nexport interface AnthropicAdapterOptions {\n  /**\n   * Anthropic model to use\n   * @default \"claude-3-5-sonnet-20241022\"\n   */\n  model?: string;\n\n  /**\n   * Maximum tokens per completion\n   * @default 4096\n   */\n  maxTokens?: number;\n\n  /**\n   * Temperature for generation (0-1)\n   * Note: Anthropic doesn't support temperature > 1\n   * @default 0\n   */\n  temperature?: number;\n\n  /**\n   * Request timeout in milliseconds\n   * @default 30000\n   */\n  timeout?: number;\n}\n\n/**\n * Creates an LLM client adapter for Anthropic Claude.\n *\n * **Setup:**\n * 1. Install Anthropic SDK: `npm install @anthropic-ai/sdk`\n * 2. Get API key from https://console.anthropic.com/\n * 3. Set environment variable: `export ANTHROPIC_API_KEY=\"sk-ant-...\"`\n *\n * **Model Options:**\n * - `claude-3-5-sonnet-20241022` - Latest, most capable (recommended)\n * - `claude-3-opus-20240229` - Most capable, expensive\n * - `claude-3-sonnet-20240229` - Balanced performance\n * - `claude-3-haiku-20240307` - Fast and affordable\n *\n * @param apiKey - Your Anthropic API key\n * @param options - Configuration options\n * @returns LLM client for use with evalsense metrics\n *\n * @example\n * ```javascript\n * // Basic usage\n * const client = createAnthropicAdapter(process.env.ANTHROPIC_API_KEY);\n * setLLMClient(client);\n *\n * // With custom model\n * const client = createAnthropicAdapter(process.env.ANTHROPIC_API_KEY, {\n *   model: 'claude-3-haiku-20240307',  // Cheaper, faster model\n *   maxTokens: 2048\n * });\n * ```\n */\nexport function createAnthropicAdapter(\n  apiKey: string,\n  options: AnthropicAdapterOptions = {}\n): LLMClient {\n  const {\n    model = \"claude-3-5-sonnet-20241022\",\n    maxTokens = 4096,\n    temperature = 0,\n    timeout = 30000,\n  } = options;\n\n  // Validate API key\n  if (!apiKey) {\n    throw new Error(\n      \"Anthropic API key is required. \" + \"Get one at https://console.anthropic.com/\"\n    );\n  }\n\n  // Validate temperature (Anthropic only supports 0-1)\n  if (temperature < 0 || temperature > 1) {\n    throw new Error(`Anthropic temperature must be between 0 and 1, got ${temperature}`);\n  }\n\n  // Lazy-load Anthropic SDK (peer dependency)\n  let Anthropic: any;\n  let anthropicClient: any;\n\n  function ensureClient() {\n    if (anthropicClient) return anthropicClient;\n\n    try {\n      // Try ESM import first\n      Anthropic = require(\"@anthropic-ai/sdk\").default || require(\"@anthropic-ai/sdk\");\n    } catch (error) {\n      throw new Error(\n        \"Anthropic SDK not found. Install it with: npm install @anthropic-ai/sdk\\n\" +\n          \"Visit https://github.com/anthropics/anthropic-sdk-typescript for documentation.\"\n      );\n    }\n\n    anthropicClient = new Anthropic({\n      apiKey,\n      timeout,\n    });\n\n    return anthropicClient;\n  }\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      const client = ensureClient();\n\n      try {\n        const message = await client.messages.create({\n          model,\n          max_tokens: maxTokens,\n          temperature,\n          messages: [{ role: \"user\", content: prompt }],\n        });\n\n        // Extract text from first content block\n        const firstBlock = message.content[0];\n        return firstBlock?.type === \"text\" ? firstBlock.text : \"\";\n      } catch (error: any) {\n        const errorMessage = error?.message || error?.error?.message || String(error);\n        throw new Error(\n          `Anthropic API error (model: ${model}): ${errorMessage}\\n` +\n            `Check your API key and usage at https://console.anthropic.com/`\n        );\n      }\n    },\n\n    async completeStructured<T>(prompt: string, schema: JSONSchema): Promise<T> {\n      // Note: Anthropic doesn't have built-in JSON mode yet\n      // We parse JSON from text response\n      const jsonPrompt =\n        prompt +\n        \"\\n\\nIMPORTANT: Respond with valid JSON only. No markdown, no explanation. \" +\n        `The JSON must match this schema: ${JSON.stringify(schema)}`;\n\n      const response = await this.complete(jsonPrompt);\n\n      try {\n        return parseJSONResponse<T>(response);\n      } catch (error: any) {\n        throw new Error(\n          `Failed to parse Anthropic response as JSON: ${error.message}\\n` +\n            `Response preview: ${response.substring(0, 200)}...`\n        );\n      }\n    },\n  };\n}\n","/**\n * Built-in OpenRouter adapter for evalsense\n *\n * OpenRouter provides access to multiple LLM providers (OpenAI, Anthropic, Google, Meta, etc.)\n * through a single unified API. Great for comparing models or avoiding vendor lock-in.\n *\n * @example\n * ```javascript\n * import { setLLMClient, createOpenRouterAdapter } from 'evalsense/metrics';\n *\n * setLLMClient(createOpenRouterAdapter(process.env.OPENROUTER_API_KEY, {\n *   model: 'anthropic/claude-3.5-sonnet'\n * }));\n * ```\n */\n\nimport type { LLMClient, JSONSchema } from \"../../../core/types.js\";\nimport { parseJSONResponse } from \"../utils.js\";\n\nexport interface OpenRouterAdapterOptions {\n  /**\n   * Model to use (in format: provider/model-name)\n   *\n   * Popular options:\n   * - `anthropic/claude-3.5-sonnet` - Latest Claude\n   * - `openai/gpt-4-turbo` - GPT-4 Turbo\n   * - `openai/gpt-3.5-turbo` - Cheap and fast\n   * - `google/gemini-pro` - Google Gemini\n   * - `meta-llama/llama-3-70b-instruct` - Open source\n   *\n   * See full list: https://openrouter.ai/models\n   *\n   * @default \"anthropic/claude-3.5-sonnet\"\n   */\n  model?: string;\n\n  /**\n   * Temperature for generation (0-2)\n   * @default 0\n   */\n  temperature?: number;\n\n  /**\n   * Maximum tokens per completion\n   * @default 4096\n   */\n  maxTokens?: number;\n\n  /**\n   * Your app name (for OpenRouter analytics)\n   * @default \"evalsense\"\n   */\n  appName?: string;\n\n  /**\n   * Your app URL (for OpenRouter analytics)\n   */\n  siteUrl?: string;\n\n  /**\n   * Request timeout in milliseconds\n   * @default 30000\n   */\n  timeout?: number;\n}\n\n/**\n * Creates an LLM client adapter for OpenRouter.\n *\n * **Setup:**\n * 1. Get API key from https://openrouter.ai/keys\n * 2. Set environment variable: `export OPENROUTER_API_KEY=\"sk-or-...\"`\n * 3. No SDK needed - uses fetch API\n *\n * **Benefits:**\n * - Access 100+ models from one API\n * - Compare different providers easily\n * - Automatic fallbacks and retries\n * - Transparent pricing\n *\n * @param apiKey - Your OpenRouter API key\n * @param options - Configuration options\n * @returns LLM client for use with evalsense metrics\n *\n * @example\n * ```javascript\n * // Basic usage\n * const client = createOpenRouterAdapter(process.env.OPENROUTER_API_KEY);\n * setLLMClient(client);\n *\n * // Use different model\n * const client = createOpenRouterAdapter(process.env.OPENROUTER_API_KEY, {\n *   model: 'openai/gpt-3.5-turbo',  // Cheaper option\n *   appName: 'my-eval-system'\n * });\n *\n * // Use free models for testing\n * const client = createOpenRouterAdapter(process.env.OPENROUTER_API_KEY, {\n *   model: 'meta-llama/llama-3-8b-instruct:free'\n * });\n * ```\n */\nexport function createOpenRouterAdapter(\n  apiKey: string,\n  options: OpenRouterAdapterOptions = {}\n): LLMClient {\n  const {\n    model = \"anthropic/claude-3.5-sonnet\",\n    temperature = 0,\n    maxTokens = 4096,\n    appName = \"evalsense\",\n    siteUrl,\n    timeout = 30000,\n  } = options;\n\n  // Validate API key\n  if (!apiKey) {\n    throw new Error(\"OpenRouter API key is required. \" + \"Get one at https://openrouter.ai/keys\");\n  }\n\n  const baseURL = \"https://openrouter.ai/api/v1\";\n\n  async function callAPI(\n    messages: Array<{ role: string; content: string }>,\n    jsonMode: boolean = false\n  ): Promise<string> {\n    const headers: Record<string, string> = {\n      Authorization: `Bearer ${apiKey}`,\n      \"Content-Type\": \"application/json\",\n      \"HTTP-Referer\": siteUrl || \"https://github.com/evalsense/evalsense\",\n      \"X-Title\": appName,\n    };\n\n    const body: any = {\n      model,\n      messages,\n      temperature,\n      max_tokens: maxTokens,\n    };\n\n    // Enable JSON mode if supported by model\n    if (jsonMode) {\n      body.response_format = { type: \"json_object\" };\n    }\n\n    const controller = new AbortController();\n    const timeoutId = setTimeout(() => controller.abort(), timeout);\n\n    try {\n      const response = await fetch(`${baseURL}/chat/completions`, {\n        method: \"POST\",\n        headers,\n        body: JSON.stringify(body),\n        signal: controller.signal,\n      });\n\n      clearTimeout(timeoutId);\n\n      if (!response.ok) {\n        const errorData: any = await response.json().catch(() => ({}));\n        const errorMessage = errorData.error?.message || response.statusText || \"Unknown error\";\n        throw new Error(`OpenRouter API error (${response.status}): ${errorMessage}`);\n      }\n\n      const data: any = await response.json();\n      return data.choices?.[0]?.message?.content ?? \"\";\n    } catch (error: any) {\n      clearTimeout(timeoutId);\n\n      if (error.name === \"AbortError\") {\n        throw new Error(`OpenRouter request timed out after ${timeout}ms (model: ${model})`);\n      }\n\n      const errorMessage = error?.message || String(error);\n      throw new Error(\n        `OpenRouter API error (model: ${model}): ${errorMessage}\\n` +\n          `Check your API key and credits at https://openrouter.ai/activity`\n      );\n    }\n  }\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      return callAPI([{ role: \"user\", content: prompt }], false);\n    },\n\n    async completeStructured<T>(prompt: string, schema: JSONSchema): Promise<T> {\n      // Try JSON mode first (works for OpenAI models via OpenRouter)\n      let response: string;\n      try {\n        response = await callAPI(\n          [{ role: \"user\", content: prompt }],\n          true // Enable JSON mode\n        );\n      } catch (error) {\n        // If JSON mode not supported, fall back to prompt engineering\n        const jsonPrompt =\n          prompt +\n          \"\\n\\nIMPORTANT: Respond with valid JSON only. No markdown, no explanation. \" +\n          `The JSON must match this schema: ${JSON.stringify(schema)}`;\n        response = await callAPI([{ role: \"user\", content: jsonPrompt }], false);\n      }\n\n      try {\n        return parseJSONResponse<T>(response);\n      } catch (error: any) {\n        throw new Error(\n          `Failed to parse OpenRouter response as JSON: ${error.message}\\n` +\n            `Model: ${model}\\n` +\n            `Response preview: ${response.substring(0, 200)}...`\n        );\n      }\n    },\n  };\n}\n"]}
+{"version":3,"sources":["../../src/metrics/custom.ts","../../src/metrics/adapters/mock.ts","../../src/metrics/index.ts"],"names":["delay"],"mappings":";;;;;AAyBO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA0D,EAAC,EACjD;AACV,EAAA,MAAM,EAAE,UAAA,GAAa,CAAA,EAAG,YAAA,GAAe,GAAE,GAAI,OAAA;AAE7C,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,QAAA,GAAW,SAAS,IAAA,CAAK,CAAC,MAAM,CAAA,CAAE,IAAA,CAAK,CAAA,CAAE,MAAM,CAAC,CAAA;AACtD,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA,EAAO,WAAW,UAAA,GAAa,YAAA;AAAA,QAC/B,KAAA,EAAO,WAAW,UAAA,GAAa;AAAA,OACjC;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;AAkBO,SAAS,mBAAA,CACd,IAAA,EACA,QAAA,EACA,OAAA,GAA2D,EAAC,EAClD;AACV,EAAA,MAAM,EAAE,aAAA,GAAgB,KAAA,EAAO,SAAA,GAAY,KAAI,GAAI,OAAA;AAEnD,EAAA,MAAM,kBAAA,GAAqB,gBAAgB,QAAA,GAAW,QAAA,CAAS,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,WAAA,EAAa,CAAA;AAEzF,EAAA,OAAO,OAAO,MAAA,KAAkD;AAC9D,IAAA,OAAO,MAAA,CAAO,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,KAAM;AAC/B,MAAA,MAAM,OAAO,aAAA,GAAgB,CAAA,CAAE,MAAA,GAAS,CAAA,CAAE,OAAO,WAAA,EAAY;AAC7D,MAAA,MAAM,OAAA,GAAU,mBAAmB,MAAA,CAAO,CAAC,MAAM,IAAA,CAAK,QAAA,CAAS,CAAC,CAAC,CAAA;AACjE,MAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,MAAA,GAAS,QAAA,CAAS,MAAA;AAExC,MAAA,OAAO;AAAA,QACL,IAAI,CAAA,CAAE,EAAA;AAAA,QACN,MAAA,EAAQ,IAAA;AAAA,QACR,KAAA;AAAA,QACA,KAAA,EAAO,KAAA,IAAS,SAAA,GAAY,UAAA,GAAa;AAAA,OAC3C;AAAA,IACF,CAAC,CAAA;AAAA,EACH,CAAA;AACF;;;ACxCO,SAAS,mBAAA,CAAoB,MAAA,GAAwB,EAAC,EAAc;AACzE,EAAA,MAAM;AAAA,IACJ,QAAA;AAAA,IACA,SAAA;AAAA,IACA,OAAAA,MAAAA,GAAQ,CAAA;AAAA,IACR,WAAA,GAAc,KAAA;AAAA,IACd,YAAA,GAAe,gBAAA;AAAA,IACf;AAAA,GACF,GAAI,MAAA;AAEJ,EAAA,IAAI,SAAA,GAAY,CAAA;AAEhB,EAAA,MAAM,cAAc,MAAwC;AAC1D,IAAA,IAAI,SAAA,IAAa,SAAA,CAAU,MAAA,GAAS,CAAA,EAAG;AACrC,MAAA,MAAM,IAAA,GAAO,UAAU,IAAA,CAAK,GAAA,CAAI,WAAW,SAAA,CAAU,MAAA,GAAS,CAAC,CAAC,CAAA;AAChE,MAAA,IAAI,CAAC,IAAA,EAAM;AACT,QAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,MAClE;AACA,MAAA,SAAA,EAAA;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAI,aAAa,MAAA,EAAW;AAC1B,MAAA,OAAO,QAAA;AAAA,IACT;AAGA,IAAA,OAAO,KAAK,SAAA,CAAU,EAAE,OAAO,GAAA,EAAK,SAAA,EAAW,iBAAiB,CAAA;AAAA,EAClE,CAAA;AAEA,EAAA,OAAO;AAAA,IACL,MAAM,SAAS,MAAA,EAAiC;AAE9C,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,OAAO,OAAO,IAAA,KAAS,QAAA,GAAW,IAAA,GAAO,IAAA,CAAK,UAAU,IAAI,CAAA;AAAA,IAC9D,CAAA;AAAA,IAEA,MAAM,kBAAA,CAAsB,MAAA,EAAgB,OAAA,EAAiC;AAE3E,MAAA,IAAI,QAAA,EAAU;AACZ,QAAA,QAAA,CAAS,MAAM,CAAA;AAAA,MACjB;AAGA,MAAA,IAAIA,SAAQ,CAAA,EAAG;AACb,QAAA,MAAM,IAAI,OAAA,CAAQ,CAAC,YAAY,UAAA,CAAW,OAAA,EAASA,MAAK,CAAC,CAAA;AAAA,MAC3D;AAGA,MAAA,IAAI,WAAA,EAAa;AACf,QAAA,MAAM,IAAI,MAAM,YAAY,CAAA;AAAA,MAC9B;AAGA,MAAA,MAAM,OAAO,WAAA,EAAY;AACzB,MAAA,IAAI,OAAO,SAAS,QAAA,EAAU;AAC5B,QAAA,IAAI;AACF,UAAA,OAAO,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,QACxB,CAAA,CAAA,MAAQ;AACN,UAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iCAAA,EAAoC,IAAI,CAAA,CAAE,CAAA;AAAA,QAC5D;AAAA,MACF;AAEA,MAAA,OAAO,IAAA;AAAA,IACT;AAAA,GACF;AACF;AAeO,SAAS,0BAAA,CACd,SAAA,EACA,OAAA,GAA8B,EAAC,EACpB;AACX,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,SAAA;AAAA,IACA,OAAO,OAAA,CAAQ;AAAA,GAChB,CAAA;AACH;AAOO,SAAS,qBAAA,CAAsB,eAAe,gBAAA,EAA6B;AAChF,EAAA,OAAO,mBAAA,CAAoB;AAAA,IACzB,WAAA,EAAa,IAAA;AAAA,IACb;AAAA,GACD,CAAA;AACH;AAcO,SAAS,oBAAoB,QAAA,EAGlC;AACA,EAAA,MAAM,UAAoB,EAAC;AAE3B,EAAA,MAAM,SAAS,mBAAA,CAAoB;AAAA,IACjC,QAAA;AAAA,IACA,QAAA,EAAU,CAAC,MAAA,KAAW,OAAA,CAAQ,KAAK,MAAM;AAAA,GAC1C,CAAA;AAED,EAAA,OAAO,EAAE,QAAQ,OAAA,EAAQ;AAC3B;;;ACxEO,IAAM,OAAA,GAAU;AAAA;AAAA,EAErB,OAAO,MAAM;AACX,IAAA,cAAA,EAAe;AACf,IAAA,aAAA,EAAc;AAAA,EAChB,CAAA;AAAA;AAAA,EAGA,IAAA,EAAM,mBAAA;AAAA;AAAA,EAGN,UAAA,EAAY,aAAA;AAAA;AAAA,EAGZ,cAAA,EAAgB,0BAAA;AAAA;AAAA,EAGhB,SAAA,EAAW,qBAAA;AAAA;AAAA,EAGX,OAAA,EAAS;AACX","file":"index.js","sourcesContent":["/**\n * Custom metric utilities\n *\n * Provides simple pattern-based and keyword-based metrics for non-LLM use cases.\n * For LLM-based custom metrics, use createLLMMetric() from evalsense/metrics.\n */\n\nimport type { MetricFn, MetricOutput, MetricConfig } from \"../core/types.js\";\n\n/**\n * Creates a simple string-matching metric\n *\n * @example\n * ```ts\n * const containsCodeMetric = createPatternMetric(\"contains-code\", [\n *   /```[\\s\\S]*?```/,\n *   /function\\s+\\w+\\s*\\(/,\n *   /const\\s+\\w+\\s*=/,\n * ]);\n *\n * const results = await containsCodeMetric({\n *   outputs: [{ id: \"1\", output: \"const x = 5\" }]\n * });\n * ```\n */\nexport function createPatternMetric(\n  name: string,\n  patterns: RegExp[],\n  options: { matchScore?: number; noMatchScore?: number } = {}\n): MetricFn {\n  const { matchScore = 1, noMatchScore = 0 } = options;\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const hasMatch = patterns.some((p) => p.test(o.output));\n      return {\n        id: o.id,\n        metric: name,\n        score: hasMatch ? matchScore : noMatchScore,\n        label: hasMatch ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n\n/**\n * Creates a keyword-based metric\n *\n * @example\n * ```ts\n * const techTermsMetric = createKeywordMetric(\"tech-terms\", [\n *   \"machine learning\",\n *   \"neural network\",\n *   \"algorithm\",\n * ], { threshold: 0.3 });\n *\n * const results = await techTermsMetric({\n *   outputs: [{ id: \"1\", output: \"This uses a neural network algorithm.\" }]\n * });\n * ```\n */\nexport function createKeywordMetric(\n  name: string,\n  keywords: string[],\n  options: { caseSensitive?: boolean; threshold?: number } = {}\n): MetricFn {\n  const { caseSensitive = false, threshold = 0.5 } = options;\n\n  const normalizedKeywords = caseSensitive ? keywords : keywords.map((k) => k.toLowerCase());\n\n  return async (config: MetricConfig): Promise<MetricOutput[]> => {\n    return config.outputs.map((o) => {\n      const text = caseSensitive ? o.output : o.output.toLowerCase();\n      const matches = normalizedKeywords.filter((k) => text.includes(k));\n      const score = matches.length / keywords.length;\n\n      return {\n        id: o.id,\n        metric: name,\n        score,\n        label: score >= threshold ? \"detected\" : \"not_detected\",\n      };\n    });\n  };\n}\n","/**\n * Mock LLM client for testing\n *\n * Provides a configurable mock implementation of LLMClient for unit tests.\n */\n\nimport type { LLMClient, JSONSchema } from \"../../core/types.js\";\n\n/**\n * Configuration for mock LLM client\n */\nexport interface MockLLMConfig {\n  /** Fixed response to return (can be string or object for JSON mode) */\n  response?: string | Record<string, unknown>;\n\n  /** Multiple responses for sequential calls */\n  responses?: Array<string | Record<string, unknown>>;\n\n  /** Delay in milliseconds before responding */\n  delay?: number;\n\n  /** Whether to throw an error */\n  shouldError?: boolean;\n\n  /** Error message to throw */\n  errorMessage?: string;\n\n  /** Function to validate prompts */\n  onPrompt?: (prompt: string) => void;\n}\n\n/**\n * Creates a mock LLM client for testing\n *\n * @example\n * ```ts\n * const mock = createMockLLMClient({\n *   response: JSON.stringify({ score: 0.8, reasoning: \"test\" }),\n *   delay: 100\n * });\n *\n * setLLMClient(mock);\n * ```\n */\nexport function createMockLLMClient(config: MockLLMConfig = {}): LLMClient {\n  const {\n    response,\n    responses,\n    delay = 0,\n    shouldError = false,\n    errorMessage = \"Mock LLM error\",\n    onPrompt,\n  } = config;\n\n  let callCount = 0;\n\n  const getResponse = (): string | Record<string, unknown> => {\n    if (responses && responses.length > 0) {\n      const resp = responses[Math.min(callCount, responses.length - 1)];\n      if (!resp) {\n        return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n      }\n      callCount++;\n      return resp;\n    }\n\n    if (response !== undefined) {\n      return response;\n    }\n\n    // Default response\n    return JSON.stringify({ score: 0.5, reasoning: \"Mock response\" });\n  };\n\n  return {\n    async complete(prompt: string): Promise<string> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response\n      const resp = getResponse();\n      return typeof resp === \"string\" ? resp : JSON.stringify(resp);\n    },\n\n    async completeStructured<T>(prompt: string, _schema: JSONSchema): Promise<T> {\n      // Call validation hook if provided\n      if (onPrompt) {\n        onPrompt(prompt);\n      }\n\n      // Simulate delay\n      if (delay > 0) {\n        await new Promise((resolve) => setTimeout(resolve, delay));\n      }\n\n      // Simulate error\n      if (shouldError) {\n        throw new Error(errorMessage);\n      }\n\n      // Return response as object\n      const resp = getResponse();\n      if (typeof resp === \"string\") {\n        try {\n          return JSON.parse(resp) as T;\n        } catch {\n          throw new Error(`Mock response is not valid JSON: ${resp}`);\n        }\n      }\n\n      return resp as T;\n    },\n  };\n}\n\n/**\n * Creates a mock client that returns sequential responses\n *\n * Useful for testing multiple calls with different responses.\n *\n * @example\n * ```ts\n * const mock = createSequentialMockClient([\n *   { score: 0.2, reasoning: \"First call\" },\n *   { score: 0.8, reasoning: \"Second call\" }\n * ]);\n * ```\n */\nexport function createSequentialMockClient(\n  responses: Array<string | Record<string, unknown>>,\n  options: { delay?: number } = {}\n): LLMClient {\n  return createMockLLMClient({\n    responses,\n    delay: options.delay,\n  });\n}\n\n/**\n * Creates a mock client that always errors\n *\n * Useful for testing error handling.\n */\nexport function createErrorMockClient(errorMessage = \"Mock LLM error\"): LLMClient {\n  return createMockLLMClient({\n    shouldError: true,\n    errorMessage,\n  });\n}\n\n/**\n * Creates a spy mock client that records all prompts\n *\n * Useful for testing what prompts are being sent to the LLM.\n *\n * @example\n * ```ts\n * const { client, prompts } = createSpyMockClient({ score: 0.5 });\n * await metric({ outputs, context, llmClient: client });\n * console.log(prompts); // See all prompts that were sent\n * ```\n */\nexport function createSpyMockClient(response: string | Record<string, unknown>): {\n  client: LLMClient;\n  prompts: string[];\n} {\n  const prompts: string[] = [];\n\n  const client = createMockLLMClient({\n    response,\n    onPrompt: (prompt) => prompts.push(prompt),\n  });\n\n  return { client, prompts };\n}\n","/**\n * Metrics module - entry point for evalsense/metrics\n *\n * Provides LLM-based metrics, custom metric utilities, and LLM client management.\n */\n\n// Re-export opinionated metrics\nexport * from \"./opinionated/index.js\";\n\n// Re-export createLLMMetric factory and types\nexport { createLLMMetric } from \"./create-metric.js\";\nexport type {\n  LLMMetric,\n  LLMMetricConfig,\n  LLMMetricOptions,\n  EvalRecord,\n  InputSpec,\n  LabelThreshold,\n  ResponseFieldType,\n} from \"./types.js\";\n\n// Re-export custom metric utilities (pattern and keyword based)\nexport { createPatternMetric, createKeywordMetric } from \"./custom.js\";\n\n// Re-export utilities\nexport {\n  normalizeScore,\n  scoreToLabel,\n  createMetricOutput,\n  BINARY_THRESHOLDS,\n  SEVERITY_THRESHOLDS,\n  batch,\n  delay,\n} from \"./utils.js\";\n\n// Re-export LLM client management\nexport {\n  setLLMClient,\n  getLLMClient,\n  resetLLMClient,\n  requireLLMClient,\n  withLLMClient,\n  configureLLM,\n  setDefaults,\n  getDefaults,\n  resetDefaults,\n} from \"./client.js\";\nexport type {\n  LLMDefaults,\n  LLMProvider,\n  ConfigureLLMOptions,\n  ConfigureLLMAutoOptions,\n} from \"./client.js\";\n\n// Re-export LLM utilities\nexport {\n  fillPrompt,\n  parseJSONResponse,\n  validateResponse,\n  extractScore,\n  createJSONSchema,\n  batchItems,\n  createLLMError,\n  withTimeout,\n} from \"./llm-utils.js\";\n\n// Re-export mock adapter for testing\nexport {\n  createMockLLMClient,\n  createSequentialMockClient,\n  createErrorMockClient,\n  createSpyMockClient,\n} from \"./adapters/mock.js\";\n\n// Re-export built-in LLM provider adapters\nexport { createOpenAIAdapter } from \"./adapters/openai.js\";\nexport { createAnthropicAdapter } from \"./adapters/anthropic.js\";\nexport { createOpenRouterAdapter } from \"./adapters/openrouter.js\";\n\n// Re-export adapter types\nexport type { OpenAIAdapterOptions } from \"./adapters/openai.js\";\nexport type { AnthropicAdapterOptions } from \"./adapters/anthropic.js\";\nexport type { OpenRouterAdapterOptions } from \"./adapters/openrouter.js\";\n\n// Testing namespace - groups all test utilities for convenient access\nimport { resetLLMClient, withLLMClient, resetDefaults } from \"./client.js\";\nimport {\n  createMockLLMClient,\n  createSequentialMockClient,\n  createErrorMockClient,\n  createSpyMockClient,\n} from \"./adapters/mock.js\";\n\n/**\n * Testing utilities for LLM metrics\n *\n * Provides convenient access to all testing-related functions in one namespace.\n *\n * @example\n * ```ts\n * import { testing } from \"evalsense/metrics\";\n *\n * describe(\"My tests\", () => {\n *   beforeEach(testing.reset);\n *\n *   it(\"test with mock\", async () => {\n *     const result = await testing.withClient(\n *       testing.mock({ response: { score: 0.8 } }),\n *       async () => hallucination([...])\n *     );\n *   });\n * });\n * ```\n */\nexport const testing = {\n  /** Resets global LLM client and defaults */\n  reset: () => {\n    resetLLMClient();\n    resetDefaults();\n  },\n\n  /** Creates a mock LLM client */\n  mock: createMockLLMClient,\n\n  /** Executes function with scoped LLM client */\n  withClient: withLLMClient,\n\n  /** Creates a mock client that returns sequential responses */\n  sequentialMock: createSequentialMockClient,\n\n  /** Creates a mock client that always errors */\n  errorMock: createErrorMockClient,\n\n  /** Creates a spy mock client that records all prompts */\n  spyMock: createSpyMockClient,\n};\n"]}

package/dist/metrics/opinionated/index.cjs

@@ -1,25 +1,25 @@
 'use strict';
 
-var chunkRZFLCWTW_cjs = require('../../chunk-RZFLCWTW.cjs');
+var chunkNCCQRZ2Y_cjs = require('../../chunk-NCCQRZ2Y.cjs');
 require('../../chunk-JEQ2X3Z6.cjs');
 
 
 
 Object.defineProperty(exports, "faithfulness", {
   enumerable: true,
-  get: function () { return chunkRZFLCWTW_cjs.faithfulness; }
+  get: function () { return chunkNCCQRZ2Y_cjs.faithfulness; }
 });
 Object.defineProperty(exports, "hallucination", {
   enumerable: true,
-  get: function () { return chunkRZFLCWTW_cjs.hallucination; }
+  get: function () { return chunkNCCQRZ2Y_cjs.hallucination; }
 });
 Object.defineProperty(exports, "relevance", {
   enumerable: true,
-  get: function () { return chunkRZFLCWTW_cjs.relevance; }
+  get: function () { return chunkNCCQRZ2Y_cjs.relevance; }
 });
 Object.defineProperty(exports, "toxicity", {
   enumerable: true,
-  get: function () { return chunkRZFLCWTW_cjs.toxicity; }
+  get: function () { return chunkNCCQRZ2Y_cjs.toxicity; }
 });
 //# sourceMappingURL=index.cjs.map
 //# sourceMappingURL=index.cjs.map