@cogitator-ai/evals 0.9.0 → 0.10.0

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export declare const VERSION: string;
 export { EvalSuite, isLLMMetric } from './eval-suite.js';
-export type { EvalTarget, EvalTargetContext, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
+export type { EvalTarget, EvalTargetContext, EvalTurn, RetrievalToolResult, EvalProgress, EvalRunOptions, EvalSuiteOptions, EvalSuiteResult, EvalSuiteStats, } from './eval-suite.js';
 export { EvalComparison } from './eval-comparison.js';
 export type { EvalComparisonOptions, EvalComparisonRunOptions, MetricComparison, ComparisonResult, } from './eval-comparison.js';
 export { EvalBuilder } from './eval-builder.js';
@@ -8,25 +8,30 @@ export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
 export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
-export type { LLMMetricFn, JudgeContext } from './metrics/llm-judge.js';
-export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export type { LLMMetricFn, JudgeContext, JudgeMessage } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
+export type { PassAtKOptions, PassAtKMetricFn } from './metrics/statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
+export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCallsScore, TrajectoryOrder, } from './metrics/tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
+export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
-export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
-export type { CustomMetricConfig } from './metrics/custom.js';
+export type { MetricFn, MetricScore, MetricUsage, MetricOptions, EvalCaseResult, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, StatisticalMetricFn, StatisticalScore, } from './metrics/types.js';
+export type { CustomMetricConfig, CustomMetricData } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';
 export { assertion } from './assertions/index.js';
 export { maxErrorRate } from './assertions/index.js';
 export type { AssertionFn, AssertionResult, AggregatedMetric, EvalStats } from './assertions/index.js';
 export { report } from './reporters/index.js';
-export type { ReporterType, ReporterOptions } from './reporters/index.js';
+export type { ReporterType, ReporterOptions, ReportedCase } from './reporters/index.js';
 export { pairedTTest } from './stats/t-test.js';
 export type { TTestResult } from './stats/t-test.js';
 export { mcnemarsTest } from './stats/mcnemar.js';
 export type { McNemarResult } from './stats/mcnemar.js';
 export { mean, median, stdDev, percentile, aggregate } from './stats/percentiles.js';
-export type { EvalCase, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
+export type { EvalCase, ExpectedToolCall, EvalSuiteConfig, JudgeConfig, JudgeConfigInput, JudgeCache, JudgeCogitator, EvalComparisonConfig, } from './schema.js';
 export { createRunEvalTool, evalTools } from './tools.js';
 export type { EvalTool } from './tools.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACtD,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,QAAQ,EACR,mBAAmB,EACnB,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,eAAe,EACf,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EACV,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnF,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC/F,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,YAAY,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,YAAY,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9F,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/E,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAElF,YAAY,EACV,QAAQ,EACR,gBAAgB,EAChB,eAAe,EACf,WAAW,EACX,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACvD,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC"}

package/dist/index.js

@@ -8,8 +8,11 @@ export { Dataset } from './datasets/index.js';
 export { loadJsonl } from './datasets/index.js';
 export { loadCsv } from './datasets/index.js';
 export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './metrics/deterministic.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './metrics/llm-judge.js';
-export { latency, cost, tokenUsage } from './metrics/statistical.js';
+export { faithfulness, relevance, coherence, helpfulness, llmMetric, contextRelevance, bindJudgeContext, } from './metrics/llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './metrics/statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './metrics/tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages, } from './metrics/retrieval.js';
+export { passageMatches } from './metrics/matching.js';
 export { metric } from './metrics/custom.js';
 export { threshold } from './assertions/index.js';
 export { noRegression } from './assertions/index.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAUtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAY1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAYlF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAYtD,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAQ/F,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,GAClB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAiB1C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAalF,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC"}

package/dist/matchers.d.ts

@@ -0,0 +1,68 @@
+import type { TrajectoryOrder } from './metrics/tool-calls.js';
+/** The matchers `evalMatchers` adds to Vitest's `expect`, typed for Vitest 5 */
+export interface EvalMatchers<R = unknown> {
+    /**
+     * The case result called the tool `name`, over any of its turns, with arguments holding `args`
+     * when given (other arguments may be there too)
+     */
+    toHaveCalledTool(name: string, args?: Record<string, unknown>): R;
+    /**
+     * The case result called the tools in `names`. `order: 'exact'` (default): those calls and
+     * no others, in this order. `'in-order'`: in this order, other calls may come between.
+     * `'any-order'`: each of them, in any order, other calls allowed
+     */
+    toFollowTrajectory(names: readonly string[], options?: {
+        order?: TrajectoryOrder;
+    }): R;
+    /**
+     * The case result retrieved a passage with this id or this text, or whose text contains this
+     * one or is contained in it when the contained text has at least three words
+     */
+    toHaveRetrieved(idOrText: string): R;
+    /**
+     * A case result scored at least `min` on the metric, or a suite result's mean of the metric
+     * is at least `min`
+     */
+    toScoreAtLeast(metricName: string, min: number): R;
+}
+declare module 'vitest' {
+    interface Assertion<R extends void | Promise<void> = void, T = unknown> extends EvalMatchers<R> {
+    }
+    interface AsymmetricMatchersContaining extends EvalMatchers {
+    }
+}
+/** The part of Vitest's matcher context the eval matchers use */
+interface MatcherContext {
+    isNot: boolean;
+}
+interface MatcherResult {
+    pass: boolean;
+    message: () => string;
+    actual?: unknown;
+    expected?: unknown;
+}
+/**
+ * Vitest matchers for eval results. Register them once, in a test file or a setup file:
+ *
+ * @example
+ * ```ts
+ * import { expect } from 'vitest';
+ * import { evalMatchers } from '@cogitator-ai/evals/matchers';
+ *
+ * expect.extend(evalMatchers);
+ *
+ * const { results } = await suite.run();
+ * expect(results[0]).toHaveCalledTool('get_weather', { city: 'Paris' });
+ * expect(results[0]).toFollowTrajectory(['search', 'answer'], { order: 'in-order' });
+ * ```
+ */
+export declare const evalMatchers: {
+    toHaveCalledTool(this: MatcherContext, received: unknown, name: string, args?: Record<string, unknown>): MatcherResult;
+    toFollowTrajectory(this: MatcherContext, received: unknown, names: readonly string[], options?: {
+        order?: TrajectoryOrder;
+    }): MatcherResult;
+    toHaveRetrieved(this: MatcherContext, received: unknown, idOrText: string): MatcherResult;
+    toScoreAtLeast(this: MatcherContext, received: unknown, metricName: string, min: number): MatcherResult;
+};
+export {};
+//# sourceMappingURL=matchers.d.ts.map

package/dist/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAS5D,gFAAgF;AAChF,MAAM,WAAW,YAAY,CAAC,CAAC,GAAG,OAAO;IACvC;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAClE;;;;OAIG;IACH,kBAAkB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,eAAe,CAAA;KAAE,GAAG,CAAC,CAAC;IACvF;;;OAGG;IACH,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC;CACpD;AAED,OAAO,QAAQ,QAAQ,CAAC;IAEtB,UAAU,SAAS,CAAC,CAAC,SAAS,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,OAAO,CAAE,SAAQ,YAAY,CAAC,CAAC,CAAC;KAAG;IAElG,UAAU,4BAA6B,SAAQ,YAAY;KAAG;CAC/D;AAED,iEAAiE;AACjE,UAAU,cAAc;IACtB,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,UAAU,aAAa;IACrB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AA4BD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,YAAY;2BAEf,cAAc,YACV,OAAO,QACX,MAAM,SACL,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,aAAa;6BAgBR,cAAc,YACV,OAAO,SACV,SAAS,MAAM,EAAE,YACf;QAAE,KAAK,CAAC,EAAE,eAAe,CAAA;KAAE,GACnC,aAAa;0BAmBM,cAAc,YAAY,OAAO,YAAY,MAAM,GAAG,aAAa;yBAejF,cAAc,YACV,OAAO,cACL,MAAM,OACb,MAAM,GACV,aAAa;CAUjB,CAAC"}

package/dist/matchers.js

@@ -0,0 +1,112 @@
+import { trajectoryScore } from './metrics/tool-calls.js';
+import { isSubset, longestCommonSubsequence, multisetOverlap, passageMatches, } from './metrics/matching.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function caseResultOf(received, matcher) {
+    if (isRecord(received) && isRecord(received.case) && typeof received.output === 'string') {
+        return received;
+    }
+    throw new TypeError(`${matcher} expects a case result, one of suite.run()'s results`);
+}
+function callText(name, args) {
+    return args === undefined ? name : `${name}(${JSON.stringify(args)})`;
+}
+function callsText(result) {
+    const calls = result.toolCalls ?? [];
+    return calls.length === 0
+        ? 'no tool calls'
+        : calls.map((c) => callText(c.name, c.arguments)).join(', ');
+}
+function negation(context) {
+    return context.isNot ? 'not ' : '';
+}
+/**
+ * Vitest matchers for eval results. Register them once, in a test file or a setup file:
+ *
+ * @example
+ * ```ts
+ * import { expect } from 'vitest';
+ * import { evalMatchers } from '@cogitator-ai/evals/matchers';
+ *
+ * expect.extend(evalMatchers);
+ *
+ * const { results } = await suite.run();
+ * expect(results[0]).toHaveCalledTool('get_weather', { city: 'Paris' });
+ * expect(results[0]).toFollowTrajectory(['search', 'answer'], { order: 'in-order' });
+ * ```
+ */
+export const evalMatchers = {
+    toHaveCalledTool(received, name, args) {
+        const result = caseResultOf(received, 'toHaveCalledTool');
+        const calls = result.toolCalls ?? [];
+        const pass = calls.some((call) => call.name === name && (args === undefined || isSubset(args, call.arguments)));
+        return {
+            pass,
+            message: () => `expected the case ${negation(this)}to call ${callText(name, args)}, it made ${callsText(result)}`,
+            actual: calls.map((c) => ({ name: c.name, arguments: c.arguments })),
+            expected: { name, ...(args !== undefined && { arguments: args }) },
+        };
+    },
+    toFollowTrajectory(received, names, options = {}) {
+        const result = caseResultOf(received, 'toFollowTrajectory');
+        const order = options.order ?? 'exact';
+        const actual = (result.toolCalls ?? []).map((c) => c.name);
+        const pass = order === 'exact'
+            ? trajectoryScore(names, actual, 'exact') === 1
+            : order === 'in-order'
+                ? longestCommonSubsequence(names, actual) === names.length
+                : multisetOverlap(names, actual) === names.length;
+        return {
+            pass,
+            message: () => `expected the tool calls ${negation(this)}to follow [${names.join(', ')}] (${order}), they were [${actual.join(', ')}]`,
+            actual,
+            expected: [...names],
+        };
+    },
+    toHaveRetrieved(received, idOrText) {
+        const result = caseResultOf(received, 'toHaveRetrieved');
+        const retrieved = result.retrieved ?? [];
+        const pass = retrieved.some((p) => passageMatches(p, idOrText));
+        const shown = retrieved.map((p) => p.id ?? JSON.stringify(p.content.slice(0, 60)));
+        return {
+            pass,
+            message: () => `expected the case ${negation(this)}to retrieve ${JSON.stringify(idOrText)}, it retrieved ${retrieved.length === 0 ? 'nothing' : `${retrieved.length} passage(s): ${shown.join(', ')}`}`,
+        };
+    },
+    toScoreAtLeast(received, metricName, min) {
+        const { score, of } = metricScoreOf(received, metricName);
+        const pass = score >= min;
+        return {
+            pass,
+            message: () => `expected ${of} ${negation(this)}to be at least ${min}, it is ${score}`,
+            actual: score,
+            expected: min,
+        };
+    },
+};
+/** The score a matcher compares: a case result's score of the metric, or a suite's mean of it */
+function metricScoreOf(received, metricName) {
+    if (isRecord(received) && isRecord(received.aggregated)) {
+        const aggregated = received.aggregated;
+        const metric = aggregated[metricName];
+        if (!metric) {
+            throw new Error(`toScoreAtLeast: the suite has no metric '${metricName}', it has ${namesText(Object.keys(aggregated))}`);
+        }
+        return { score: metric.mean, of: `the mean of ${metricName}` };
+    }
+    const result = caseResultOf(received, 'toScoreAtLeast');
+    const scores = result.scores ?? [];
+    const found = scores.find((s) => s.name === metricName);
+    if (!found) {
+        throw new Error(`toScoreAtLeast: the case has no score for '${metricName}', it has ${namesText(scores.map((s) => s.name))}`);
+    }
+    return {
+        score: found.score,
+        of: `the ${metricName} score of the case${found.details ? ` (${found.details})` : ''}`,
+    };
+}
+function namesText(names) {
+    return names.length === 0 ? 'none' : names.map((n) => `'${n}'`).join(', ');
+}
+//# sourceMappingURL=matchers.js.map

package/dist/matchers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EACL,QAAQ,EACR,wBAAwB,EACxB,eAAe,EACf,cAAc,GACf,MAAM,oBAAoB,CAAC;AA8C5B,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AACrD,CAAC;AAED,SAAS,YAAY,CAAC,QAAiB,EAAE,OAAe;IACtD,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzF,OAAO,QAAqC,CAAC;IAC/C,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,GAAG,OAAO,sDAAsD,CAAC,CAAC;AACxF,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,IAA8B;IAC5D,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,MAAsB;IACvC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;QACvB,CAAC,CAAC,eAAe;QACjB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,OAAuB;IACvC,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,gBAAgB,CAEd,QAAiB,EACjB,IAAY,EACZ,IAA8B;QAE9B,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,kBAAkB,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CACrB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CACvF,CAAC;QACF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,qBAAqB,QAAQ,CAAC,IAAI,CAAC,WAAW,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,SAAS,CAAC,MAAM,CAAC,EAAE;YACpG,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YACpE,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,EAAE;SACnE,CAAC;IACJ,CAAC;IAED,kBAAkB,CAEhB,QAAiB,EACjB,KAAwB,EACxB,UAAuC,EAAE;QAEzC,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,IAAI,GACR,KAAK,KAAK,OAAO;YACf,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;YAC/C,CAAC,CAAC,KAAK,KAAK,UAAU;gBACpB,CAAC,CAAC,wBAAwB,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM;gBAC1D,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC;QACxD,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,2BAA2B,QAAQ,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,iBAAiB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACzH,MAAM;YACN,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC;SACrB,CAAC;IACJ,CAAC;IAED,eAAe,CAAuB,QAAiB,EAAE,QAAgB;QACvE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QAChE,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACnF,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CACZ,qBAAqB,QAAQ,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,kBACxE,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,gBAAgB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAC1F,EAAE;SACL,CAAC;IACJ,CAAC;IAED,cAAc,CAEZ,QAAiB,EACjB,UAAkB,EAClB,GAAW;QAEX,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC;QAC1B,OAAO;YACL,IAAI;YACJ,OAAO,EAAE,GAAG,EAAE,CAAC,YAAY,EAAE,IAAI,QAAQ,CAAC,IAAI,CAAC,kBAAkB,GAAG,WAAW,KAAK,EAAE;YACtF,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,GAAG;SACd,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,iGAAiG;AACjG,SAAS,aAAa,CAAC,QAAiB,EAAE,UAAkB;IAC1D,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,MAAM,UAAU,GAAG,QAAQ,CAAC,UAA2C,CAAC;QACxE,MAAM,MAAM,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CACb,4CAA4C,UAAU,aAAa,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CACxG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,eAAe,UAAU,EAAE,EAAE,CAAC;IACjE,CAAC;IACD,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IACxD,MAAM,MAAM,GAA4B,MAAqC,CAAC,MAAM,IAAI,EAAE,CAAC;IAC3F,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IACxD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CACb,8CAA8C,UAAU,aAAa,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAC5G,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,EAAE,EAAE,OAAO,UAAU,qBAAqB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;KACvF,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAe;IAChC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7E,CAAC"}

package/dist/metrics/custom.d.ts

@@ -1,12 +1,7 @@
-import type { MetricFn } from './types.js';
+import type { MetricFn, EvalCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from './types.js';
 export interface CustomMetricConfig {
     name: string;
-    evaluate: (data: {
-        input: string;
-        output: string;
-        expected?: string;
-        context?: Record<string, unknown>;
-    }) => Promise<{
+    evaluate: (data: CustomMetricData) => Promise<{
         score: number;
         details?: string;
     }> | {
@@ -14,6 +9,23 @@ export interface CustomMetricConfig {
         details?: string;
     };
 }
+/** What a custom metric's `evaluate` gets about a case */
+export interface CustomMetricData {
+    input: string;
+    output: string;
+    expected?: string;
+    context?: Record<string, unknown>;
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the run retrieved */
+    retrieved?: readonly Passage[];
+    /** Each turn of a multi-turn case */
+    turns?: readonly EvalTurnResult[];
+    /** The whole case result, the case itself included */
+    result: EvalCaseResult;
+}
 /**
  * A metric from an `evaluate` function. Its score is clamped to 0 - 1. When `evaluate` throws or
  * returns a score that is not a finite number (such as NaN from a division by zero), the metric

package/dist/metrics/custom.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,SAAS,CAAC;AAGxD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CAAC,IAAI,EAAE;QACf,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC1F;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CAsC3D"}
+{"version":3,"file":"custom.d.ts","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EACd,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACR,MAAM,SAAS,CAAC;AAGjB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,CACR,IAAI,EAAE,gBAAgB,KACnB,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CACzF;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,sDAAsD;IACtD,MAAM,EAAE,cAAc,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,kBAAkB,GAAG,QAAQ,CA2C3D"}

package/dist/metrics/custom.js

@@ -12,6 +12,11 @@ export function metric(config) {
                 output: result.output,
                 expected: result.case.expected,
                 context: result.case.context,
+                toolCalls: result.toolCalls,
+                toolResults: result.toolResults,
+                retrieved: result.retrieved,
+                turns: result.turns,
+                result,
             });
             const invalid = nonFiniteScoreError(score);
             if (invalid) {

package/dist/metrics/custom.js.map

@@ -1 +1 @@
-{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AAY9C;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;gBACrC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}
+{"version":3,"file":"custom.js","sourceRoot":"","sources":["../../src/metrics/custom.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,mBAAmB,EAAE,MAAM,SAAS,CAAC;AA2B9C;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,MAA0B;IAC/C,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,MAAsB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBAC/C,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;gBACxB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ;gBAC9B,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO;gBAC5B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;aACP,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,CAAC;oBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;oBACrC,KAAK,EAAE,OAAO;iBACf,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACtC,GAAG,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,CAAC;aAC1C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,mBAAmB,OAAO,EAAE;gBACrC,KAAK,EAAE,OAAO;aACf,CAAC;QACJ,CAAC;IACH,CAAC,CAAa,CAAC;IAEf,EAAE,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;IAC5B,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/metrics/index.d.ts

@@ -1,8 +1,12 @@
-export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, } from './types.js';
+export type { MetricScore, MetricUsage, MetricOptions, EvalCaseResult, MetricFn, StatisticalMetricFn, StatisticalScore, ScoredCaseResult, EvalToolCall, EvalToolResult, EvalTurnResult, Passage, } from './types.js';
 export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
-export type { CustomMetricConfig } from './custom.js';
-export { latency, cost, tokenUsage } from './statistical.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
+export type { CustomMetricConfig, CustomMetricData } from './custom.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './statistical.js';
+export type { PassAtKOptions, PassAtKMetricFn } from './statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './tool-calls.js';
+export type { ToolCallsOptions, TrajectoryOptions, ToolArgumentsMatch, ToolCallsScore, TrajectoryOrder, } from './tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages } from './retrieval.js';
+export { faithfulness, relevance, coherence, helpfulness, contextRelevance, llmMetric, bindJudgeContext, } from './llm-judge.js';
 export type { JudgeContext, LLMMetricFn } from './llm-judge.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/metrics/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,GACjB,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,WAAW,EACX,WAAW,EACX,aAAa,EACb,cAAc,EACd,QAAQ,EACR,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,GACR,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAClC,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACrE,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACvF,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,eAAe,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC"}

package/dist/metrics/index.js

@@ -1,5 +1,7 @@
 export { exactMatch, contains, containsAll, containsAny, similarity, regex, jsonSchema, } from './deterministic.js';
 export { metric } from './custom.js';
-export { latency, cost, tokenUsage } from './statistical.js';
-export { faithfulness, relevance, coherence, helpfulness, llmMetric, bindJudgeContext, } from './llm-judge.js';
+export { latency, cost, tokenUsage, passAtK, passAtKEstimate, isPassAtKMetric, } from './statistical.js';
+export { toolCalls, trajectory, toolCallMatches, trajectoryScore } from './tool-calls.js';
+export { retrievalRecall, retrievalPrecision, retrievalMRR, ragSearchPassages } from './retrieval.js';
+export { faithfulness, relevance, coherence, helpfulness, contextRelevance, llmMetric, bindJudgeContext, } from './llm-judge.js';
 //# sourceMappingURL=index.js.map

package/dist/metrics/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/metrics/index.ts"],"names":[],"mappings":"AAcA,OAAO,EACL,UAAU,EACV,QAAQ,EACR,WAAW,EACX,WAAW,EACX,UAAU,EACV,KAAK,EACL,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,OAAO,EACL,OAAO,EACP,IAAI,EACJ,UAAU,EACV,OAAO,EACP,eAAe,EACf,eAAe,GAChB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQvF,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACnG,OAAO,EACL,YAAY,EACZ,SAAS,EACT,SAAS,EACT,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,gBAAgB,GACjB,MAAM,aAAa,CAAC"}

package/dist/metrics/llm-judge.d.ts

@@ -1,5 +1,5 @@
 import type { JudgeCogitator, JudgeConfig } from '../schema.js';
-import type { MetricFn, MetricOptions, MetricUsage } from './types.js';
+import type { MetricFn, EvalCaseResult, MetricOptions, MetricUsage } from './types.js';
 export interface JudgeContext {
     cogitator: {
         run: (opts: {
@@ -11,10 +11,18 @@ export interface JudgeContext {
     };
     judgeConfig: JudgeConfig;
 }
+/** What the judge is asked about a case, or why the case is not judged and scores 0 */
+export type JudgeMessage = {
+    message: string;
+} | {
+    skip: string;
+};
 export interface LLMMetricFn extends MetricFn {
     readonly requiresJudge: true;
     readonly __judgeSystemPrompt: string;
     readonly __judgeName: string;
+    /** Builds what the judge reads about a case. Default: input, context, expected and response */
+    readonly __judgeMessage?: (result: EvalCaseResult) => JudgeMessage;
 }
 /**
  * A judge context that runs the judge as a Cogitator agent on `judgeConfig.model`,
@@ -22,7 +30,18 @@ export interface LLMMetricFn extends MetricFn {
  */
 export declare function judgeContextFor(cogitator: JudgeCogitator, judgeConfig: JudgeConfig): JudgeContext;
 export declare function bindJudgeContext(metric: LLMMetricFn, context: JudgeContext): MetricFn;
+/**
+ * How faithful the response is to the input and the evidence: the passages the run retrieved
+ * when the case has any, its `context` otherwise.
+ */
 export declare function faithfulness(opts?: MetricOptions): LLMMetricFn;
+/**
+ * Whether the passages the run retrieved are relevant to the input, judged on the input (the
+ * conversation for a multi-turn case) and the passages alone (the case's `context` when nothing
+ * was retrieved). A case with neither scores 0
+ * without a judge call.
+ */
+export declare function contextRelevance(opts?: MetricOptions): LLMMetricFn;
 export declare function relevance(opts?: MetricOptions): LLMMetricFn;
 export declare function coherence(opts?: MetricOptions): LLMMetricFn;
 export declare function helpfulness(opts?: MetricOptions): LLMMetricFn;

package/dist/metrics/llm-judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAA+B,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAEjG,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAyDjG;AA0GD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA6CrF;AAED,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK9D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}
+{"version":3,"file":"llm-judge.d.ts","sourceRoot":"","sources":["../../src/metrics/llm-judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,cAAc,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,KAAK,EACV,QAAQ,EACR,cAAc,EAEd,aAAa,EACb,WAAW,EAEZ,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QACT,GAAG,EAAE,CAAC,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,OAAO,CAAC;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,WAAW,CAAA;SAAE,CAAC,CAAC;KACpF,CAAC;IACF,WAAW,EAAE,WAAW,CAAC;CAC1B;AAED,uFAAuF;AACvF,MAAM,MAAM,YAAY,GAAG;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAElE,MAAM,WAAW,WAAY,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,aAAa,EAAE,IAAI,CAAC;IAC7B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+FAA+F;IAC/F,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,YAAY,CAAC;CACpE;AAsFD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,GAAG,YAAY,CAyDjG;AAgJD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,GAAG,QAAQ,CA8CrF;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAM9D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAUlE;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,SAAS,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK3D;AAED,wBAAgB,WAAW,CAAC,IAAI,CAAC,EAAE,aAAa,GAAG,WAAW,CAK7D;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAK7E"}

package/dist/metrics/llm-judge.js

@@ -195,7 +195,7 @@ function limiter(limit) {
         }
     };
 }
-function createJudgeMetric(name, systemPrompt) {
+function createJudgeMetric(name, systemPrompt, message) {
     const fn = (async (_result) => {
         return { name, score: 0, details: 'unbound judge metric — call bindJudgeContext first' };
     });
@@ -203,6 +203,8 @@ function createJudgeMetric(name, systemPrompt) {
     Object.defineProperty(fn, 'requiresJudge', { value: true, writable: false });
     Object.defineProperty(fn, '__judgeSystemPrompt', { value: systemPrompt, writable: false });
     Object.defineProperty(fn, '__judgeName', { value: name, writable: false });
+    if (message)
+        Object.defineProperty(fn, '__judgeMessage', { value: message, writable: false });
     return fn;
 }
 function contextText(context) {
@@ -210,23 +212,57 @@ function contextText(context) {
         .map(([key, value]) => `${key}: ${typeof value === 'string' ? value : JSON.stringify(value, null, 2)}`)
         .join('\n');
 }
+function passagesText(passages) {
+    return passages
+        .map((p, i) => `[${i + 1}]${p.source ? ` (${p.source})` : ''} ${p.content}`)
+        .join('\n');
+}
 /**
- * What the judge reads: the input, the case's context when it has one (source documents, a
- * dossier, retrieved passages), the expected answer and the response.
+ * The evidence of a case: its retrieved passages when it has any and `preferRetrieved` is set,
+ * else its context, else nothing
  */
-function buildUserMessage(result) {
-    const expected = result.case.expected || 'N/A';
+function evidenceBlock(result, preferRetrieved) {
+    if (preferRetrieved && result.retrieved && result.retrieved.length > 0) {
+        return `Retrieved passages:\n${passagesText(result.retrieved)}`;
+    }
     const context = result.case.context;
-    const contextBlock = context && Object.keys(context).length > 0 ? `\nContext:\n${contextText(context)}` : '';
-    return `Input: ${result.case.input}${contextBlock}\nExpected: ${expected}\nResponse: ${result.output}`;
+    return context && Object.keys(context).length > 0
+        ? `Context:\n${contextText(context)}`
+        : undefined;
+}
+/**
+ * What the response answers. A single-turn case: its input. A multi-turn case: the conversation
+ * before the last user message, each turn's input and answer, then that last message marked as
+ * the one the response replies to.
+ */
+function inputBlock(result) {
+    const inputs = [result.case.input, ...(result.case.turns ?? [])];
+    if (inputs.length === 1)
+        return `Input: ${result.case.input}`;
+    const earlier = inputs
+        .slice(0, -1)
+        .map((input, i) => `User: ${input}\nAssistant: ${result.turns?.[i]?.output ?? ''}`);
+    return `Conversation so far:\n${earlier.join('\n')}\nInput (the last user message, which the response answers): ${inputs[inputs.length - 1]}`;
+}
+/**
+ * What the judge reads: the input (the conversation for a multi-turn case), the case's evidence when it has some (its context, or the
+ * retrieved passages for metrics that prefer them), the expected answer and the response.
+ */
+function buildUserMessage(result, preferRetrieved = false) {
+    const expected = result.case.expected || 'N/A';
+    const evidence = evidenceBlock(result, preferRetrieved);
+    const evidenceText = evidence === undefined ? '' : `\n${evidence}`;
+    return `${inputBlock(result)}${evidenceText}\nExpected: ${expected}\nResponse: ${result.output}`;
 }
 export function bindJudgeContext(metric, context) {
     const name = metric.__judgeName;
     const systemPrompt = metric.__judgeSystemPrompt;
     const bound = (async (result) => {
         try {
-            const userMessage = buildUserMessage(result);
-            const prompt = `${systemPrompt}\n\n${userMessage}`;
+            const message = metric.__judgeMessage?.(result) ?? { message: buildUserMessage(result) };
+            if ('skip' in message)
+                return { name, score: 0, details: message.skip };
+            const prompt = `${systemPrompt}\n\n${message.message}`;
             const runResult = await context.cogitator.run({ input: prompt });
             const usage = usageOf(runResult.usage);
             const parsed = parseJudgeOutput(runResult.output);
@@ -260,8 +296,26 @@ export function bindJudgeContext(metric, context) {
     Object.defineProperty(bound, 'metricName', { value: name, writable: false });
     return bound;
 }
+/**
+ * How faithful the response is to the input and the evidence: the passages the run retrieved
+ * when the case has any, its `context` otherwise.
+ */
 export function faithfulness(opts) {
-    return createJudgeMetric(opts?.name ?? 'faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input, the context when there is one, and the response, rate how faithful the response is to the facts and information in the input and the context. Claims that neither supports count against it.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');
+    return createJudgeMetric(opts?.name ?? 'faithfulness', 'You are evaluating the faithfulness of an AI assistant\'s response.\n\nGiven the input, the context or retrieved passages when there are some, and the response, rate how faithful the response is to the facts and information in the input and the context. Claims that neither supports count against it.\n\nScore from 0.0 (completely unfaithful) to 1.0 (perfectly faithful).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}', (result) => ({ message: buildUserMessage(result, true) }));
+}
+/**
+ * Whether the passages the run retrieved are relevant to the input, judged on the input (the
+ * conversation for a multi-turn case) and the passages alone (the case's `context` when nothing
+ * was retrieved). A case with neither scores 0
+ * without a judge call.
+ */
+export function contextRelevance(opts) {
+    return createJudgeMetric(opts?.name ?? 'contextRelevance', 'You are evaluating the context an AI assistant retrieved to answer a request.\n\nGiven the input and the retrieved passages (or context), rate how relevant they are to the input: whether they hold the information needed to answer it, with little that is off topic.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}', (result) => {
+        const evidence = evidenceBlock(result, true);
+        if (evidence === undefined)
+            return { skip: 'no retrieved passages or context to judge' };
+        return { message: `${inputBlock(result)}\n${evidence}` };
+    });
 }
 export function relevance(opts) {
     return createJudgeMetric(opts?.name ?? 'relevance', 'You are evaluating the relevance of an AI assistant\'s response.\n\nGiven the input and the response, rate how relevant the response is to the question asked.\n\nScore from 0.0 (completely irrelevant) to 1.0 (perfectly relevant).\n\nRespond with JSON: {"score": <number>, "reasoning": "<explanation>"}');