@cogitator-ai/evals 0.9.0 → 0.11.0

package/dist/metrics/index.js

@@ -1,5 +1,7 @@
-export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
+export { exactMatch, contains, containsAll, containsAny, similarity, tokenF1, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
-export { latency, cost, tokenUsage } from './statistical.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages } from './retrieval.js';
+export { faithfulness, relevance, coherence, helpfulness, contextRelevance, llmMetric, bindJudgeContext, } from './llm-judge.js';
 //# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAcA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,OAAO,EACP,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQvF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}

package/dist/metrics/llm-judge.d.ts

@@ -1,5 +1,5 @@
 import type { JudgeCogitator, JudgeConfig } from '../schema.js';
-import type { MetricFn, MetricOptions, MetricUsage } from './types.js';
+import type { MetricFn, EvalCaseResult, MetricOptions, MetricUsage } from './types.js';
 export interface JudgeContext {
     cogitator: {
         run: (opts: {
@@ -11,10 +11,18 @@ export interface JudgeContext {
     };
     judgeConfig: JudgeConfig;
 }
+/** What the judge is asked about a case, or why the case is not judged and scores 0 */
+export type JudgeMessage = {
+    message: string;
+} | {
+    skip: string;
+};
 export interface LLMMetricFn extends MetricFn {
     readonly requiresJudge: true;
     readonly __judgeSystemPrompt: string;
     readonly __judgeName: string;
+    /** Builds what the judge reads about a case. Default: input, context, expected and response */
+    readonly __judgeMessage?: (result: EvalCaseResult) => JudgeMessage;
 }
 /**
  * A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
@@ -22,7 +30,18 @@ export interface LLMMetricFn extends MetricFn {
  */
 export declare function judgeContextFor(cogitator: JudgeCogitator, judgeConfig: JudgeConfig): JudgeContext;
 export declare function bindJudgeContext(metric: LLMMetricFn, context: JudgeContext): MetricFn;
+/**
+ * How faithful the response is to the input and the evidence: the passages the run retrieved
+ * when the case has any, its `context` otherwise.
+ */
 export declare function faithfulness(opts?: MetricOptions): LLMMetricFn;
+/**
+ * Whether the passages the run retrieved are relevant to the input, judged on the input (the
+ * conversation for a multi-turn case) and the passages alone (the case's `context` when nothing
+ * was retrieved). A case with neither scores 0
+ * without a judge call.
+ */
+export declare function contextRelevance(opts?: MetricOptions): LLMMetricFn;
 export declare function relevance(opts?: MetricOptions): LLMMetricFn;
 export declare function coherence(opts?: MetricOptions): LLMMetricFn;
 export declare function helpfulness(opts?: MetricOptions): LLMMetricFn;

package/dist/metrics/llm-judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAA+B,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEjG,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAyDjG;AA0GD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA6CrF;AAED,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK9D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EAEd,aAAa,EACb,WAAW,EAEZ,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,uFAAuF;AACvF,MAAM,MAAM,YAAY,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAElE,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+FAA+F;IAC/F,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,YAAY,CAAC;CACpE;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAyDjG;AAgJD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA8CrF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAM9D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAUlE;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}

package/dist/metrics/llm-judge.js

@@ -195,7 +195,7 @@ function limiter(limit) {
         }
     };
 }
-function createJudgeMetric(name, systemPrompt) {
+function createJudgeMetric(name, systemPrompt, message) {
     const fn = (async (_result) => {
         return { name, score: 0, details: 'unbound judge metric — call bindJudgeContext first' };
     });
@@ -203,6 +203,8 @@ function createJudgeMetric(name, systemPrompt) {
     Object.defineProperty(fn, 'requiresJudge', { value: true, writable: false });
     Object.defineProperty(fn, '__judgeSystemPrompt', { value: systemPrompt, writable: false });
     Object.defineProperty(fn, '__judgeName', { value: name, writable: false });
+    if (message)
+        Object.defineProperty(fn, '__judgeMessage', { value: message, writable: false });
     return fn;
 }
 function contextText(context) {
@@ -210,23 +212,57 @@ function contextText(context) {
         .map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}`)
         .join('\n');
 }
+function passagesText(passages) {
+    return passages
+        .map((p, i) => `[${i + 1}]${p.source ? ` (${p.source})` : ''} ${p.content}`)
+        .join('\n');
+}
 /**
- * What the judge reads: the input, the case's context when it has one (source documents, a
- * dossier, retrieved passages), the expected answer and the response.
+ * The evidence of a case: its retrieved passages when it has any and `preferRetrieved` is set,
+ * else its context, else nothing
  */
-function buildUserMessage(result) {
-    const expected = result.case.expected || 'N/A';
+function evidenceBlock(result, preferRetrieved) {
+    if (preferRetrieved && result.retrieved && result.retrieved.length > 0) {
+        return `Retrieved passages:\n${passagesText(result.retrieved)}`;
+    }
     const context = result.case.context;
-    const contextBlock = context && Object.keys(context).length > 0 ? `\nContext:\n${contextText(context)}` : '';
-    return `Input: ${result.case.input}${contextBlock}\nExpected: ${expected}\nResponse: ${result.output}`;
+    return context && Object.keys(context).length > 0
+        ? `Context:\n${contextText(context)}`
+        : undefined;
+}
+/**
+ * What the response answers. A single-turn case: its input. A multi-turn case: the conversation
+ * before the last user message, each turn's input and answer, then that last message marked as
+ * the one the response replies to.
+ */
+function inputBlock(result) {
+    const inputs = [result.case.input, ...(result.case.turns ?? [])];
+    if (inputs.length === 1)
+        return `Input: ${result.case.input}`;
+    const earlier = inputs
+        .slice(0, -1)
+        .map((input, i) => `User: ${input}\nAssistant: ${result.turns?.[i]?.output ?? ''}`);
+    return `Conversation so far:\n${earlier.join('\n')}\nInput (the last user message, which the response answers): ${inputs[inputs.length - 1]}`;
+}
+/**
+ * What the judge reads: the input (the conversation for a multi-turn case), the case's evidence when it has some (its context, or the
+ * retrieved passages for metrics that prefer them), the expected answer and the response.
+ */
+function buildUserMessage(result, preferRetrieved = false) {
+    const expected = result.case.expected || 'N/A';
+    const evidence = evidenceBlock(result, preferRetrieved);
+    const evidenceText = evidence === undefined ? '' : `\n${evidence}`;
+    return `${inputBlock(result)}${evidenceText}\nExpected: ${expected}\nResponse: ${result.output}`;
 }
 export function bindJudgeContext(metric, context) {
     const name = metric.__judgeName;
     const systemPrompt = metric.__judgeSystemPrompt;
     const bound = (async (result) => {
         try {
-            const userMessage = buildUserMessage(result);
-            const prompt = `${systemPrompt}\n\n${userMessage}`;
+            const message = metric.__judgeMessage?.(result) ?? { message: buildUserMessage(result) };
+            if ('skip' in message)
+                return { name, score: 0, details: message.skip };
+            const prompt = `${systemPrompt}\n\n${message.message}`;
             const runResult = await context.cogitator.run({ input: prompt });
             const usage = usageOf(runResult.usage);
             const parsed = parseJudgeOutput(runResult.output);
@@ -260,8 +296,26 @@ export function bindJudgeContext(metric, context) {
     Object.defineProperty(bound, 'metricName', { value: name, writable: false });
     return bound;
 }
+/**
+ * How faithful the response is to the input and the evidence: the passages the run retrieved
+ * when the case has any, its `context` otherwise.
+ */
 export function faithfulness(opts) {
-    return createJudgeMetric(opts?.name ?? 'faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input, the context when there is one, and the response, rate how faithful the response is to the facts and information in the input and the context. Claims that neither supports count against it.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+    return createJudgeMetric(opts?.name ?? 'faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input, the context or retrieved passages when there are some, and the response, rate how faithful the response is to the facts and information in the input and the context. Claims that neither supports count against it.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}', (result) => ({ message: buildUserMessage(result, true) }));
+}
+/**
+ * Whether the passages the run retrieved are relevant to the input, judged on the input (the
+ * conversation for a multi-turn case) and the passages alone (the case's `context` when nothing
+ * was retrieved). A case with neither scores 0
+ * without a judge call.
+ */
+export function contextRelevance(opts) {
+    return createJudgeMetric(opts?.name ?? 'contextRelevance', 'You are evaluating the context an AI assistant retrieved to answer a request.\n\nGiven the input and the retrieved passages (or context), rate how relevant they are to the input: whether they hold the information needed to answer it, with little that is off topic.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}', (result) => {
+        const evidence = evidenceBlock(result, true);
+        if (evidence === undefined)
+            return { skip: 'no retrieved passages or context to judge' };
+        return { message: `${inputBlock(result)}\n${evidence}` };
+    });
 }
 export function relevance(opts) {
     return createJudgeMetric(opts?.name ?? 'relevance', 'You are evaluating the relevance of an AI assistant\'s response.\n\nGiven the input and the response, rate how relevant the response is to the question asked.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');

package/dist/metrics/llm-judge.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAiBxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI,SAAS,CAAC;IAC1F,MAAM,KAAK,GACT,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,MAAM,KAAK,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;YAC3F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4D,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAM,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,UAAU,CAAC,KAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtF,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,WAAW;QACX,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;KAC5E,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,SAAS,CAAC,KAAiB,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAY,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAW,EAAE,OAAe;IACvE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,KAAc;IACzD,OAAO,CAAC,WAAW,CACjB,6BAA6B,MAAM,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1G,EAAE,IAAI,EAAE,6BAA6B,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,SAAS,QAAQ,CAAC,WAAwB,EAAE,KAAa;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,iBAAiB;QACjB,kBAAkB;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,SAAS,IAAI,IAAI;QAC7B,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;;YAC7E,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACZ,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,YAAoB;IAC3D,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB;IAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,MAAM,YAAY,GAChB,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACzG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,WAAW,EAAE,CAAC;YAEnD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,qaAAqa,CACta,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"llm-judge.js","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AA6BxB,MAAM,YAAY,GAAG,6BAA6B,CAAC;AACnD,MAAM,WAAW,GAAG,aAAa,CAAC;AAClC,MAAM,cAAc,GAClB,yFAAyF,CAAC;AAE5F,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE5E,MAAM,kBAAkB,GACtB,oKAAoK,CAAC;AAEvK,MAAM,WAAW,GAAG,gCAAgC,CAAC;AAOrD,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IACvE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,KAAiD,CAAC;IAC/E,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,sGAAsG;AACtG,SAAS,cAAc,CAAC,GAAW;IACjC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,6FAA6F;AAC7F,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAkC,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,SAAS,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QAChD,IAAI,OAAO;YAAE,OAAO,OAAO,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACjC,OAAO,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,KAAgC,CAAC;IAC1F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC;QAChC,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,SAAyB,EAAE,WAAwB;IACjF,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,IAAI,SAAS,CAAC;IAC1F,MAAM,KAAK,GACT,WAAW,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,UAAwC,CAAC;IAC7C,MAAM,KAAK,GAAG,GAAG,EAAE,CACjB,CAAC,UAAU,KAAK,MAAM,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAC/C,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CACZ,IAAI,KAAK,CAAC;QACR,IAAI,EAAE,YAAY;QAClB,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,GAAG,CAAC,WAAW,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,CAAC;QAChF,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE;KAC9D,CAAC,EACJ,CAAC,KAAc,EAAE,EAAE;QACjB,MAAM,IAAI,KAAK,CACb,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpG,CAAC;IACJ,CAAC,CACF,CAAC,CAAC;IAEL,MAAM,KAAK,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,MAAM,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM;YAC3F,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4D,CAAC;IAErF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QAClD,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAM,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,UAAU,CAAC,KAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtF,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,KAAK,EAAE,KAAa,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,MAAM,EAAE,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3B,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IAEF,OAAO;QACL,WAAW;QACX,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE;KAC5E,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,SAAS,CAAC,KAAiB,EAAE,GAAW;IACrD,IAAI,CAAC;QACH,MAAM,GAAG,GAAY,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzB,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,UAAU,CAAC,KAAiB,EAAE,GAAW,EAAE,OAAe;IACvE,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,KAAc;IACzD,OAAO,CAAC,WAAW,CACjB,6BAA6B,MAAM,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAC1G,EAAE,IAAI,EAAE,6BAA6B,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B;;;GAGG;AACH,SAAS,QAAQ,CAAC,WAAwB,EAAE,KAAa;IACvD,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,iBAAiB;QACjB,kBAAkB;QAClB,WAAW,CAAC,KAAK;QACjB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,SAAS,IAAI,IAAI;QAC7B,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED,wGAAwG;AACxG,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,OAAO,KAAK,EAAK,IAAsB,EAAc,EAAE;QACrD,IAAI,OAAO,IAAI,KAAK;YAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;;YAC7E,OAAO,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACZ,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CACxB,IAAY,EACZ,YAAoB,EACpB,OAAkD;IAElD,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,OAAuB,EAAwB,EAAE;QAClE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,oDAAoD,EAAE,CAAC;IAC3F,CAAC,CAAgB,CAAC;IAElB,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,qBAAqB,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO;QAAE,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,gBAAgB,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE9F,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,OAAgC;IACnD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CACF,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CACf,GAAG,GAAG,KAAK,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAClF;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,QAA4B;IAChD,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;SAC3E,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,MAAsB,EAAE,eAAwB;IACrE,IAAI,eAAe,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvE,OAAO,wBAAwB,YAAY,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;IAClE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;IACpC,OAAO,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;QAC/C,CAAC,CAAC,aAAa,WAAW,CAAC,OAAO,CAAC,EAAE;QACrC,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAsB;IACxC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM;SACnB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACZ,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,KAAK,gBAAgB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,EAAE,CAAC,CAAC;IACtF,OAAO,yBAAyB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,gEAAgE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;AAChJ,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAAsB,EAAE,eAAe,GAAG,KAAK;IACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;IAC/C,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;IACnE,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,YAAY,eAAe,QAAQ,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAmB,EAAE,OAAqB;IACzE,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAEhD,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAwB,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YACzF,IAAI,MAAM,IAAI,OAAO;gBAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,GAAG,YAAY,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;YAEvD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAElD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI;oBACJ,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,WAAW;oBACpB,KAAK,EAAE,WAAW;oBAClB,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;iBACxB,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAEvD,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,OAAO;gBACd,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpE,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;aACxB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI;gBACJ,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,gBAAgB,OAAO,EAAE;gBAClC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;IAE7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,cAAc,EAC5B,6bAA6b,EAC7b,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAC1D,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAoB;IACnD,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,kBAAkB,EAChC,yZAAyZ,EACzZ,CAAC,MAAM,EAAE,EAAE;QACT,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,2CAA2C,EAAE,CAAC;QACzF,OAAO,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,EAAE,CAAC;IAC3D,CAAC,CACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,+SAA+S,CAChT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAoB;IAC5C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,WAAW,EACzB,uTAAuT,CACxT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,OAAO,iBAAiB,CACtB,IAAI,EAAE,IAAI,IAAI,aAAa,EAC3B,qTAAqT,CACtT,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAsC;IAC9D,OAAO,iBAAiB,CACtB,IAAI,CAAC,IAAI,EACT,GAAG,IAAI,CAAC,MAAM,oGAAoG,CACnH,CAAC;AACJ,CAAC"}

package/dist/metrics/matching.d.ts

@@ -0,0 +1,28 @@
+import type { Passage } from './types.js';
+/** Whether two JSON-like values are equal, objects key by key and arrays item by item */
+export declare function deepEqual(a: unknown, b: unknown): boolean;
+/**
+ * Whether `actual` holds everything `expected` has: every key of an expected object is in the
+ * actual one with a matching value, arrays match item by item with the same length, and other
+ * values are equal
+ */
+export declare function isSubset(expected: unknown, actual: unknown): boolean;
+/** Text compared without case, punctuation or runs of whitespace */
+export declare function normalizeText(text: string): string;
+/**
+ * Whether a retrieved passage is the expected one: the same id, the same words once case and
+ * punctuation are set aside, or texts where one holds the other's words in a row when that one
+ * has at least three words, so a short id or a single word never matches every passage that
+ * happens to hold it
+ */
+export declare function passageMatches(passage: Passage, expected: string): boolean;
+/**
+ * A largest one-to-one matching between `left` and `right` items, `matches` telling which pairs
+ * may match (Kuhn's augmenting paths): the index of the right item each matched left item took
+ */
+export declare function maxMatching<L, R>(left: readonly L[], right: readonly R[], matches: (l: L, r: R) => boolean): Map<number, number>;
+/** The length of the longest common subsequence of two sequences */
+export declare function longestCommonSubsequence(a: readonly string[], b: readonly string[]): number;
+/** How many items two multisets share */
+export declare function multisetOverlap(a: readonly string[], b: readonly string[]): number;
+//# sourceMappingURL=matching.d.ts.map

package/dist/metrics/matching.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matching.d.ts","sourceRoot":"","sources":["../../src/metrics/matching.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAMvC,yFAAyF;AACzF,wBAAgB,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,GAAG,OAAO,CAWzD;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO,CAYpE;AAED,oEAAoE;AACpE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMlD;AAcD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQ1E;AAED;;;GAGG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,EAC9B,IAAI,EAAE,SAAS,CAAC,EAAE,EAClB,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,OAAO,GAC/B,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAuBrB;AAED,oEAAoE;AACpE,wBAAgB,wBAAwB,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAU3F;AAED,yCAAyC;AACzC,wBAAgB,eAAe,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAYlF"}

package/dist/metrics/matching.js

@@ -0,0 +1,124 @@
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+/** Whether two JSON-like values are equal, objects key by key and arrays item by item */
+export function deepEqual(a, b) {
+    if (Object.is(a, b))
+        return true;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        return a.length === b.length && a.every((item, i) => deepEqual(item, b[i]));
+    }
+    if (isRecord(a) && isRecord(b)) {
+        const keys = Object.keys(a);
+        if (keys.length !== Object.keys(b).length)
+            return false;
+        return keys.every((key) => Object.hasOwn(b, key) && deepEqual(a[key], b[key]));
+    }
+    return false;
+}
+/**
+ * Whether `actual` holds everything `expected` has: every key of an expected object is in the
+ * actual one with a matching value, arrays match item by item with the same length, and other
+ * values are equal
+ */
+export function isSubset(expected, actual) {
+    if (Array.isArray(expected) && Array.isArray(actual)) {
+        return (expected.length === actual.length && expected.every((item, i) => isSubset(item, actual[i])));
+    }
+    if (isRecord(expected) && isRecord(actual)) {
+        return Object.entries(expected).every(([key, value]) => Object.hasOwn(actual, key) && isSubset(value, actual[key]));
+    }
+    return Object.is(expected, actual);
+}
+/** Text compared without case, punctuation or runs of whitespace */
+export function normalizeText(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+/** Texts shorter than this many words match only as a whole, never as a part of another */
+const MIN_CONTAINED_WORDS = 3;
+function wordCount(text) {
+    return text === '' ? 0 : text.split(' ').length;
+}
+/** Whether `inner` is a run of whole words of `outer`, both normalized */
+function containsWords(outer, inner) {
+    return ` ${outer} `.includes(` ${inner} `);
+}
+/**
+ * Whether a retrieved passage is the expected one: the same id, the same words once case and
+ * punctuation are set aside, or texts where one holds the other's words in a row when that one
+ * has at least three words, so a short id or a single word never matches every passage that
+ * happens to hold it
+ */
+export function passageMatches(passage, expected) {
+    if (passage.id !== undefined && passage.id === expected)
+        return true;
+    const content = normalizeText(passage.content);
+    const target = normalizeText(expected);
+    if (content === '' || target === '')
+        return false;
+    if (content === target)
+        return true;
+    if (wordCount(target) >= MIN_CONTAINED_WORDS && containsWords(content, target))
+        return true;
+    return wordCount(content) >= MIN_CONTAINED_WORDS && containsWords(target, content);
+}
+/**
+ * A largest one-to-one matching between `left` and `right` items, `matches` telling which pairs
+ * may match (Kuhn's augmenting paths): the index of the right item each matched left item took
+ */
+export function maxMatching(left, right, matches) {
+    const owner = new Array(right.length).fill(-1);
+    const edges = left.map((l) => right.flatMap((r, j) => (matches(l, r) ? [j] : [])));
+    const augment = (i, seen) => {
+        for (const j of edges[i]) {
+            if (seen[j])
+                continue;
+            seen[j] = true;
+            if (owner[j] === -1 || augment(owner[j], seen)) {
+                owner[j] = i;
+                return true;
+            }
+        }
+        return false;
+    };
+    for (let i = 0; i < left.length; i++)
+        augment(i, new Array(right.length).fill(false));
+    const pairs = new Map();
+    owner.forEach((i, j) => {
+        if (i !== -1)
+            pairs.set(i, j);
+    });
+    return pairs;
+}
+/** The length of the longest common subsequence of two sequences */
+export function longestCommonSubsequence(a, b) {
+    let prev = new Array(b.length + 1).fill(0);
+    for (const item of a) {
+        const curr = new Array(b.length + 1).fill(0);
+        for (let j = 1; j <= b.length; j++) {
+            curr[j] = item === b[j - 1] ? prev[j - 1] + 1 : Math.max(prev[j], curr[j - 1]);
+        }
+        prev = curr;
+    }
+    return prev[b.length];
+}
+/** How many items two multisets share */
+export function multisetOverlap(a, b) {
+    const counts = new Map();
+    for (const item of a)
+        counts.set(item, (counts.get(item) ?? 0) + 1);
+    let shared = 0;
+    for (const item of b) {
+        const left = counts.get(item) ?? 0;
+        if (left > 0) {
+            counts.set(item, left - 1);
+            shared++;
+        }
+    }
+    return shared;
+}
+//# sourceMappingURL=matching.js.map

package/dist/metrics/matching.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matching.js","sourceRoot":"","sources":["../../src/metrics/matching.ts"],"names":[],"mappings":"AAEA,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,SAAS,CAAC,CAAU,EAAE,CAAU;IAC9C,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,QAAQ,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACxD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,QAAiB,EAAE,MAAe;IACzD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACrD,OAAO,CACL,QAAQ,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAC5F,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3C,OAAO,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CACnC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AACrC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC;SACjC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,2FAA2F;AAC3F,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAE9B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;AAClD,CAAC;AAED,0EAA0E;AAC1E,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa;IACjD,OAAO,IAAI,KAAK,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,OAAgB,EAAE,QAAgB;IAC/D,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,IAAI,OAAO,CAAC,EAAE,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACrE,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,OAAO,KAAK,EAAE,IAAI,MAAM,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAClD,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,mBAAmB,IAAI,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5F,OAAO,SAAS,CAAC,OAAO,CAAC,IAAI,mBAAmB,IAAI,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACrF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,IAAkB,EAClB,KAAmB,EACnB,OAAgC;IAEhC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEnF,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,IAAe,EAAW,EAAE;QACtD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,CAAC,CAAC;gBAAE,SAAS;YACtB,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC/C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBACb,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC,EAAE,IAAI,KAAK,CAAU,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE/F,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,IAAI,CAAC,KAAK,CAAC,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,wBAAwB,CAAC,CAAoB,EAAE,CAAoB;IACjF,IAAI,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,IAAI,KAAK,CAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AACxB,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,eAAe,CAAC,CAAoB,EAAE,CAAoB;IACxE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,CAAC;QAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC3B,MAAM,EAAE,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/metrics/retrieval.d.ts

@@ -0,0 +1,21 @@
+import type { MetricFn, MetricOptions, Passage } from './types.js';
+/**
+ * The passages of a tool result shaped like the `rag_search` tool's of `@cogitator-ai/rag`:
+ * `{ results: [{ content, chunkId?, score?, source?, ... }] }`, each result's `id` (or else its
+ * `chunkId`) as the passage id. Undefined for any other result.
+ */
+export declare function ragSearchPassages(toolResult: {
+    name: string;
+    result: unknown;
+}): Passage[] | undefined;
+/** The share of the expected passages that some retrieved passage matches */
+export declare function retrievalRecall(opts?: MetricOptions): MetricFn;
+/** The share of the retrieved passages that match an expected one */
+export declare function retrievalPrecision(opts?: MetricOptions): MetricFn;
+/**
+ * The reciprocal rank of the first retrieved passage that matches an expected one: 1 when it
+ * came first, 1/2 second and so on, 0 when none matches. Passages rank in the order they came,
+ * one tool result after the other.
+ */
+export declare function retrievalMRR(opts?: MetricOptions): MetricFn;
+//# sourceMappingURL=retrieval.d.ts.map

package/dist/metrics/retrieval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval.d.ts","sourceRoot":"","sources":["../../src/metrics/retrieval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAuBhE;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,UAAU,EAAE;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;CACjB,GAAG,OAAO,EAAE,GAAG,SAAS,CAOxB;AA6CD,6EAA6E;AAC7E,wBAAgB,eAAe,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAW9D;AAED,qEAAqE;AACrE,wBAAgB,kBAAkB,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAWjE;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,QAAQ,CAa3D"}

package/dist/metrics/retrieval.js

@@ -0,0 +1,108 @@
+import { passageMatches } from './matching.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function passageOf(item) {
+    if (!isRecord(item) || typeof item.content !== 'string')
+        return undefined;
+    const id = typeof item.id === 'string'
+        ? item.id
+        : typeof item.chunkId === 'string'
+            ? item.chunkId
+            : undefined;
+    return {
+        ...(id !== undefined && { id }),
+        content: item.content,
+        ...(typeof item.score === 'number' && Number.isFinite(item.score) && { score: item.score }),
+        ...(typeof item.source === 'string' && { source: item.source }),
+    };
+}
+/**
+ * The passages of a tool result shaped like the `rag_search` tool's of `@cogitator-ai/rag`:
+ * `{ results: [{ content, chunkId?, score?, source?, ... }] }`, each result's `id` (or else its
+ * `chunkId`) as the passage id. Undefined for any other result.
+ */
+export function ragSearchPassages(toolResult) {
+    const payload = toolResult.result;
+    if (!isRecord(payload) || !Array.isArray(payload.results))
+        return undefined;
+    return payload.results.flatMap((item) => {
+        const passage = passageOf(item);
+        return passage ? [passage] : [];
+    });
+}
+function createMetricFn(name, fn) {
+    fn.metricName = name;
+    return fn;
+}
+/**
+ * A retrieval metric over a case's `retrieved` passages and its `expectedPassages`. A case
+ * expecting no passages scores 1 when nothing was retrieved and 0 otherwise, and so does a case
+ * expecting passages when nothing was retrieved.
+ */
+function retrievalMetric(name, compute) {
+    return createMetricFn(name, (async (result) => {
+        const expected = result.case.expectedPassages;
+        if (expected === undefined) {
+            return { name, score: 0, details: 'no expected passages provided' };
+        }
+        const retrieved = result.retrieved ?? [];
+        if (expected.length === 0 || retrieved.length === 0) {
+            const score = expected.length === retrieved.length ? 1 : 0;
+            return {
+                name,
+                score,
+                details: score === 1
+                    ? undefined
+                    : retrieved.length === 0
+                        ? 'nothing was retrieved'
+                        : 'no passages were expected',
+            };
+        }
+        const { score, details } = compute({ expected, retrieved });
+        return { name, score, ...(details !== undefined && { details }) };
+    }));
+}
+/** The share of the expected passages that some retrieved passage matches */
+export function retrievalRecall(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalRecall', ({ expected, retrieved }) => {
+        const missing = expected.filter((e) => !retrieved.some((p) => passageMatches(p, e)));
+        return {
+            score: (expected.length - missing.length) / expected.length,
+            details: missing.length > 0
+                ? `not retrieved: ${missing.map((e) => JSON.stringify(e)).join(', ')}`
+                : undefined,
+        };
+    });
+}
+/** The share of the retrieved passages that match an expected one */
+export function retrievalPrecision(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalPrecision', ({ expected, retrieved }) => {
+        const relevant = retrieved.filter((p) => expected.some((e) => passageMatches(p, e))).length;
+        return {
+            score: relevant / retrieved.length,
+            details: relevant < retrieved.length
+                ? `${retrieved.length - relevant} of ${retrieved.length} retrieved passages were not expected`
+                : undefined,
+        };
+    });
+}
+/**
+ * The reciprocal rank of the first retrieved passage that matches an expected one: 1 when it
+ * came first, 1/2 second and so on, 0 when none matches. Passages rank in the order they came,
+ * one tool result after the other.
+ */
+export function retrievalMRR(opts) {
+    return retrievalMetric(opts?.name ?? 'retrievalMRR', ({ expected, retrieved }) => {
+        const rank = retrieved.findIndex((p) => expected.some((e) => passageMatches(p, e))) + 1;
+        return {
+            score: rank === 0 ? 0 : 1 / rank,
+            details: rank === 0
+                ? 'no retrieved passage was expected'
+                : rank > 1
+                    ? `first expected passage at rank ${rank}`
+                    : undefined,
+        };
+    });
+}
+//# sourceMappingURL=retrieval.js.map

package/dist/metrics/retrieval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval.js","sourceRoot":"","sources":["../../src/metrics/retrieval.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,SAAS,CAAC,IAAa;IAC9B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC1E,MAAM,EAAE,GACN,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ;QACzB,CAAC,CAAC,IAAI,CAAC,EAAE;QACT,CAAC,CAAC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ;YAChC,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,SAAS,CAAC;IAClB,OAAO;QACL,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,EAAE,EAAE,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,GAAG,CAAC,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3F,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,UAGjC;IACC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC5E,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAChC,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,EAAY;IAChD,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,CAAC;AACZ,CAAC;AAOD;;;;GAIG;AACH,SAAS,eAAe,CACtB,IAAY,EACZ,OAA2E;IAE3E,OAAO,cAAc,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC9C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,+BAA+B,EAAE,CAAC;QACtE,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3D,OAAO;gBACL,IAAI;gBACJ,KAAK;gBACL,OAAO,EACL,KAAK,KAAK,CAAC;oBACT,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;wBACtB,CAAC,CAAC,uBAAuB;wBACzB,CAAC,CAAC,2BAA2B;aACpC,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC5D,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IACpE,CAAC,CAAa,CAAC,CAAC;AAClB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,eAAe,CAAC,IAAoB;IAClD,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,iBAAiB,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QAClF,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACrF,OAAO;YACL,KAAK,EAAE,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;YAC3D,OAAO,EACL,OAAO,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,kBAAkB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACtE,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,qEAAqE;AACrE,MAAM,UAAU,kBAAkB,CAAC,IAAoB;IACrD,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,oBAAoB,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QACrF,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5F,OAAO;YACL,KAAK,EAAE,QAAQ,GAAG,SAAS,CAAC,MAAM;YAClC,OAAO,EACL,QAAQ,GAAG,SAAS,CAAC,MAAM;gBACzB,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,QAAQ,OAAO,SAAS,CAAC,MAAM,uCAAuC;gBAC9F,CAAC,CAAC,SAAS;SAChB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAoB;IAC/C,OAAO,eAAe,CAAC,IAAI,EAAE,IAAI,IAAI,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,EAAE;QAC/E,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxF,OAAO;YACL,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;YAChC,OAAO,EACL,IAAI,KAAK,CAAC;gBACR,CAAC,CAAC,mCAAmC;gBACrC,CAAC,CAAC,IAAI,GAAG,CAAC;oBACR,CAAC,CAAC,kCAAkC,IAAI,EAAE;oBAC1C,CAAC,CAAC,SAAS;SAClB,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/metrics/statistical.d.ts

@@ -5,4 +5,32 @@ export declare function latency(): StatisticalMetricFn;
 export declare function cost(): StatisticalMetricFn;
 /** Total tokens per case, for cases that report usage. */
 export declare function tokenUsage(): StatisticalMetricFn;
+export interface PassAtKOptions {
+    /** Trials that may be drawn, a positive integer up to the suite's `trials` */
+    k: number;
+    /** The score at which a trial passes. Default: 1 */
+    threshold?: number;
+}
+/** A `passAtK()` metric: the suite checks its metric and its `k` against its own setup */
+export type PassAtKMetricFn = StatisticalMetricFn & {
+    readonly passAtK: {
+        readonly metric: string;
+        readonly k: number;
+        readonly threshold: number;
+    };
+};
+/** Whether a statistical metric is a `passAtK()` one */
+export declare function isPassAtKMetric(metric: StatisticalMetricFn): metric is PassAtKMetricFn;
+/**
+ * The chance that at least one of `k` trials drawn from `n`, `c` of which passed, passes:
+ * 1 - C(n - c, k) / C(n, k), computed as a product so large counts do not overflow
+ */
+export declare function passAtKEstimate(n: number, c: number, k: number): number;
+/**
+ * pass@k of a per-case metric over the suite's trials: for each case, the unbiased estimate of
+ * the chance that at least one of `k` trials scores `threshold` or more, averaged over the cases
+ * the metric scored at least one trial of. A trial that errored counts as a failure. Named
+ * `passAtK(<metric>,<k>)`. The suite must run at least `k` trials and have the metric.
+ */
+export declare function passAtK(metricName: string, opts: PassAtKOptions): PassAtKMetricFn;
 //# sourceMappingURL=statistical.d.ts.map

package/dist/metrics/statistical.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBrF,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD"}
+{"version":3,"file":"statistical.d.ts","sourceRoot":"","sources":["../../src/metrics/statistical.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA+B,mBAAmB,EAAoB,MAAM,SAAS,CAAC;AAkBlG,8CAA8C;AAC9C,wBAAgB,OAAO,IAAI,mBAAmB,CAoB7C;AAED,0DAA0D;AAC1D,wBAAgB,IAAI,IAAI,mBAAmB,CAoB1C;AAED,0DAA0D;AAC1D,wBAAgB,UAAU,IAAI,mBAAmB,CAsBhD;AAED,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,CAAC,EAAE,MAAM,CAAC;IACV,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,0FAA0F;AAC1F,MAAM,MAAM,eAAe,GAAG,mBAAmB,GAAG;IAClD,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/F,CAAC;AAEF,wDAAwD;AACxD,wBAAgB,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,IAAI,eAAe,CAEtF;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKvE;AAID;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,eAAe,CA4CjF"}