@cogitator-ai/evals 0.13.2 → 0.14.0

package/dist/plugin-evals/glob.js

@@ -0,0 +1,63 @@
+const SPECIAL = /[\\^$.*+?()[\]{}|/]/;
+/**
+ * A glob as a regular expression over a whole string: `*` matches within a path segment, `**`
+ * across segments, `?` one character, `[...]` a class and `{a,b}` alternatives
+ */
+export function globToRegExp(glob) {
+    let source = '';
+    let depth = 0;
+    for (let at = 0; at < glob.length; at++) {
+        const char = glob[at];
+        if (char === '*') {
+            if (glob[at + 1] === '*') {
+                at++;
+                if (glob[at + 1] === '/') {
+                    at++;
+                    source += '(?:.*/)?';
+                }
+                else
+                    source += '.*';
+            }
+            else
+                source += '[^/]*';
+        }
+        else if (char === '?')
+            source += '[^/]';
+        else if (char === '[') {
+            const end = glob.indexOf(']', at + 2);
+            if (end === -1)
+                source += '\\[';
+            else {
+                const body = glob.slice(at + 1, end).replace(/\\/g, '\\\\');
+                source += `[${body.startsWith('!') ? `^${body.slice(1)}` : body}]`;
+                at = end;
+            }
+        }
+        else if (char === '{') {
+            depth++;
+            source += '(?:';
+        }
+        else if (char === '}' && depth > 0) {
+            depth--;
+            source += ')';
+        }
+        else if (char === ',' && depth > 0)
+            source += '|';
+        else if (char === '\\' && at + 1 < glob.length) {
+            at++;
+            source += escapeChar(glob[at]);
+        }
+        else
+            source += escapeChar(char);
+    }
+    source += ')'.repeat(depth);
+    return new RegExp(`^${source}$`);
+}
+function escapeChar(char) {
+    return SPECIAL.test(char) ? `\\${char}` : char;
+}
+/** Whether `value` matches `glob` as a whole */
+export function globMatches(glob, value) {
+    return globToRegExp(glob).test(value);
+}
+//# sourceMappingURL=glob.js.map

package/dist/plugin-evals/glob.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"glob.js","sourceRoot":"","sources":["../../src/plugin-evals/glob.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG,qBAAqB,CAAC;AAEtC;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACzB,EAAE,EAAE,CAAC;gBACL,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACzB,EAAE,EAAE,CAAC;oBACL,MAAM,IAAI,UAAU,CAAC;gBACvB,CAAC;;oBAAM,MAAM,IAAI,IAAI,CAAC;YACxB,CAAC;;gBAAM,MAAM,IAAI,OAAO,CAAC;QAC3B,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG;YAAE,MAAM,IAAI,MAAM,CAAC;aACrC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YACtC,IAAI,GAAG,KAAK,CAAC,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC;iBAC3B,CAAC;gBACJ,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5D,MAAM,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;gBACnE,EAAE,GAAG,GAAG,CAAC;YACX,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxB,KAAK,EAAE,CAAC;YACR,MAAM,IAAI,KAAK,CAAC;QAClB,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACrC,KAAK,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,CAAC;QAChB,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,GAAG,CAAC;aAC/C,IAAI,IAAI,KAAK,IAAI,IAAI,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/C,EAAE,EAAE,CAAC;YACL,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACjC,CAAC;;YAAM,MAAM,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,OAAO,IAAI,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAa;IACrD,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACxC,CAAC"}

package/dist/plugin-evals/graders.d.ts

@@ -0,0 +1,21 @@
+import { type PluginEvalJudge } from './judge.js';
+import { type TranscriptEntry } from './transcript.js';
+import type { PluginEvalGrader, PluginEvalGraderResult, PluginEvalMockCall, PluginEvalToolCall } from './types.js';
+/** What one run left for its graders */
+export interface RunCapture {
+    prompt: string;
+    lastMessage: string;
+    entries: TranscriptEntry[];
+    toolCalls: PluginEvalToolCall[];
+    mockCalls: PluginEvalMockCall[];
+    filesCreated: string[];
+    workspaceRoot: string;
+    caseDir: string;
+}
+type Verdict = Pick<PluginEvalGraderResult, 'passed' | 'explanation'> & Partial<Pick<PluginEvalGraderResult, 'votes' | 'evidence' | 'costUsd'>>;
+/** Whether a grader calls the judge, and so costs something */
+export declare function isPaidGrader(grader: PluginEvalGrader): boolean;
+/** Applies one grader to a run */
+export declare function grade(grader: PluginEvalGrader, capture: RunCapture, judge: PluginEvalJudge, signal?: AbortSignal): Promise<Verdict>;
+export {};
+//# sourceMappingURL=graders.d.ts.map

package/dist/plugin-evals/graders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/graders.ts"],"names":[],"mappings":"AAIA,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAE5D,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EACV,gBAAgB,EAChB,sBAAsB,EACtB,kBAAkB,EAElB,kBAAkB,EAGnB,MAAM,SAAS,CAAC;AAEjB,wCAAwC;AACxC,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,SAAS,EAAE,kBAAkB,EAAE,CAAC;IAChC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB;AAsED,KAAK,OAAO,GAAG,IAAI,CAAC,sBAAsB,EAAE,QAAQ,GAAG,aAAa,CAAC,GACnE,OAAO,CAAC,IAAI,CAAC,sBAAsB,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC;AAE1E,+DAA+D;AAC/D,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAE9D;AAED,kCAAkC;AAClC,wBAAsB,KAAK,CACzB,MAAM,EAAE,gBAAgB,EACxB,OAAO,EAAE,UAAU,EACnB,KAAK,EAAE,eAAe,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,OAAO,CAAC,CA8HlB"}

package/dist/plugin-evals/graders.js

@@ -0,0 +1,198 @@
+import { readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { isFile, isInside } from './files.js';
+import { globMatches } from './glob.js';
+import { JUDGE_VOTES } from './judge.js';
+import { namesCall } from './tools.js';
+import { parseTranscript, readableTranscript, traceLines, } from './transcript.js';
+/** How many messages of a trace an `llm` judge sees from its start and from its end */
+const JUDGED_TRACE_ENDS = 12;
+const BINARY_SNIFF = 8192;
+function targetLabel(target) {
+    return typeof target === 'string' ? target : `file ${target.path}`;
+}
+function contentOf(target, capture, forJudge) {
+    if (typeof target === 'object') {
+        const path = resolve(capture.workspaceRoot, target.path);
+        if (!isInside(capture.workspaceRoot, path) || !isFile(path)) {
+            return { missing: `the file ${target.path} does not exist in the workspace` };
+        }
+        const bytes = readFileSync(path);
+        const binary = bytes.subarray(0, BINARY_SNIFF).includes(0);
+        return { text: bytes.toString('utf8'), ...(binary && { binary: true }) };
+    }
+    switch (target) {
+        case 'last_message':
+            return { text: capture.lastMessage };
+        case 'files':
+            return { text: capture.filesCreated.join('\n') };
+        case 'mock_calls':
+            return { text: capture.mockCalls.map((call) => JSON.stringify(call)).join('\n') };
+        case 'trace': {
+            const lines = traceLines(capture.entries);
+            if (!forJudge || lines.length <= JUDGED_TRACE_ENDS * 2)
+                return { text: lines.join('\n') };
+            const left = lines.length - JUDGED_TRACE_ENDS * 2;
+            return {
+                text: [
+                    ...lines.slice(0, JUDGED_TRACE_ENDS),
+                    `... ${left} message${left === 1 ? '' : 's'} left out ...`,
+                    ...lines.slice(-JUDGED_TRACE_ENDS),
+                ].join('\n'),
+            };
+        }
+    }
+}
+function excerpt(text, at, length) {
+    const start = Math.max(0, at - 30);
+    const snippet = text.slice(start, at + Math.max(length, 1) + 30).replace(/\s+/g, ' ');
+    return `${start > 0 ? '...' : ''}${snippet}${at + length + 30 < text.length ? '...' : ''}`;
+}
+function matchesInput(call, inputMatch) {
+    return inputMatch === undefined || new RegExp(inputMatch).test(JSON.stringify(call.input));
+}
+function firstIndex(calls, matcher) {
+    return calls.findIndex((call) => namesCall(matcher.tool, call) && matchesInput(call, matcher.inputMatch));
+}
+function matcherText(matcher) {
+    return matcher.inputMatch === undefined
+        ? matcher.tool
+        : `${matcher.tool} /${matcher.inputMatch}/`;
+}
+function plural(count, word) {
+    return `${count} ${word}${count === 1 ? '' : 's'}`;
+}
+/** Whether a grader calls the judge, and so costs something */
+export function isPaidGrader(grader) {
+    return grader.type === 'llm' || grader.type === 'baseline';
+}
+/** Applies one grader to a run */
+export async function grade(grader, capture, judge, signal) {
+    switch (grader.type) {
+        case 'regex': {
+            const content = contentOf(grader.target, capture, false);
+            if ('missing' in content) {
+                return grader.match === 'not_contains'
+                    ? {
+                        passed: true,
+                        explanation: `${content.missing}, so it does not contain /${grader.pattern}/`,
+                    }
+                    : { passed: false, explanation: content.missing };
+            }
+            const where = targetLabel(grader.target);
+            if (grader.match.startsWith('count:')) {
+                const wanted = Number(grader.match.slice('count:'.length));
+                const count = [...content.text.matchAll(new RegExp(grader.pattern, `${grader.flags}g`))]
+                    .length;
+                return {
+                    passed: count === wanted,
+                    explanation: `/${grader.pattern}/${grader.flags} matched ${plural(count, 'time')} in ${where}, ${wanted} wanted`,
+                };
+            }
+            const found = new RegExp(grader.pattern, grader.flags).exec(content.text);
+            const contains = grader.match === 'contains';
+            if (found) {
+                return {
+                    passed: contains,
+                    explanation: `/${grader.pattern}/${grader.flags} found in ${where}: "${excerpt(content.text, found.index, found[0].length)}"`,
+                };
+            }
+            return {
+                passed: !contains,
+                explanation: `/${grader.pattern}/${grader.flags} not found in ${where}`,
+            };
+        }
+        case 'tool_used': {
+            const count = capture.toolCalls.filter((call) => namesCall(grader.tool, call) && matchesInput(call, grader.inputMatch)).length;
+            const passed = count >= grader.min && (grader.max === undefined || count <= grader.max);
+            const bounds = grader.max === undefined
+                ? `at least ${grader.min}`
+                : grader.min === grader.max
+                    ? `exactly ${grader.min}`
+                    : `${grader.min} to ${grader.max}`;
+            const filter = grader.inputMatch === undefined ? '' : ` with input matching /${grader.inputMatch}/`;
+            return {
+                passed,
+                explanation: `${grader.tool} called ${plural(count, 'time')}${filter}, ${bounds} wanted`,
+            };
+        }
+        case 'tool_order': {
+            const before = firstIndex(capture.toolCalls, grader.before);
+            const after = firstIndex(capture.toolCalls, grader.after);
+            if (before === -1)
+                return { passed: false, explanation: `${matcherText(grader.before)} was not called` };
+            if (after === -1)
+                return { passed: false, explanation: `${matcherText(grader.after)} was not called` };
+            return {
+                passed: before < after,
+                explanation: `${matcherText(grader.before)} first at call ${before + 1}, ${matcherText(grader.after)} first at call ${after + 1}`,
+            };
+        }
+        case 'file_exists': {
+            const matching = capture.filesCreated.filter((path) => globMatches(grader.path, path));
+            if (grader.exists) {
+                return matching.length > 0
+                    ? { passed: true, explanation: `created ${matching.slice(0, 5).join(', ')}` }
+                    : { passed: false, explanation: `no file created during the run matches ${grader.path}` };
+            }
+            return matching.length === 0
+                ? { passed: true, explanation: `no file created during the run matches ${grader.path}` }
+                : { passed: false, explanation: `created ${matching.slice(0, 5).join(', ')}, none wanted` };
+        }
+        case 'llm': {
+            const content = contentOf(grader.focus, capture, true);
+            if ('missing' in content)
+                return { passed: false, explanation: content.missing };
+            if (content.binary) {
+                return {
+                    passed: false,
+                    explanation: `${targetLabel(grader.focus)} is a binary file, which the judge cannot read: write it out as text and grade that`,
+                };
+            }
+            const answer = await judge.ask({
+                kind: 'llm',
+                criteria: grader.criteria,
+                evidence: content.text,
+                prompt: capture.prompt,
+            }, signal);
+            return {
+                ...tally(answer.votes),
+                votes: answer.votes,
+                evidence: content.text,
+                costUsd: answer.costUsd,
+            };
+        }
+        case 'baseline': {
+            const file = join(capture.caseDir, grader.baselineFile);
+            const reference = parseTranscript(readFileSync(file, 'utf8'));
+            if ('error' in reference) {
+                return { passed: false, explanation: `${grader.baselineFile}: ${reference.error}` };
+            }
+            const run = readableTranscript(capture.entries);
+            const answer = await judge.ask({
+                kind: 'baseline',
+                criteria: grader.criteria,
+                reference: readableTranscript(reference),
+                run,
+                prompt: capture.prompt,
+            }, signal);
+            return {
+                ...tally(answer.votes),
+                votes: answer.votes,
+                evidence: run,
+                costUsd: answer.costUsd,
+            };
+        }
+    }
+}
+function tally(votes) {
+    const pass = votes.filter((vote) => vote.verdict === 'PASS').length;
+    const passed = pass * 2 > JUDGE_VOTES;
+    const deciding = votes.find((vote) => (vote.verdict === 'PASS') === passed && vote.reasoning);
+    const reason = deciding?.reasoning ?? votes.find((vote) => vote.reasoning)?.reasoning;
+    return {
+        passed,
+        explanation: `${pass} of ${votes.length} votes PASS${reason ? `: ${reason}` : ''}`,
+    };
+}
+//# sourceMappingURL=graders.js.map

package/dist/plugin-evals/graders.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graders.js","sourceRoot":"","sources":["../../src/plugin-evals/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,UAAU,GAEX,MAAM,cAAc,CAAC;AAuBtB,uFAAuF;AACvF,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,YAAY,GAAG,IAAI,CAAC;AAI1B,SAAS,WAAW,CAAC,MAAwB;IAC3C,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,SAAS,CAAC,MAAwB,EAAE,OAAmB,EAAE,QAAiB;IACjF,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,OAAO,EAAE,OAAO,EAAE,YAAY,MAAM,CAAC,IAAI,kCAAkC,EAAE,CAAC;QAChF,CAAC;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC3E,CAAC;IACD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,cAAc;YACjB,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QACvC,KAAK,OAAO;YACV,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,KAAK,YAAY;YACf,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpF,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,iBAAiB,GAAG,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,iBAAiB,GAAG,CAAC,CAAC;YAClD,OAAO;gBACL,IAAI,EAAE;oBACJ,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC;oBACpC,OAAO,IAAI,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,eAAe;oBAC1D,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC;iBACnC,CAAC,IAAI,CAAC,IAAI,CAAC;aACb,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,IAAY,EAAE,EAAU,EAAE,MAAc;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACtF,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,GAAG,EAAE,GAAG,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;AAC7F,CAAC;AAED,SAAS,YAAY,CAAC,IAAwB,EAAE,UAA8B;IAC5E,OAAO,UAAU,KAAK,SAAS,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AAC7F,CAAC;AAED,SAAS,UAAU,CAAC,KAAoC,EAAE,OAA8B;IACtF,OAAO,KAAK,CAAC,SAAS,CACpB,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,CAClF,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,OAA8B;IACjD,OAAO,OAAO,CAAC,UAAU,KAAK,SAAS;QACrC,CAAC,CAAC,OAAO,CAAC,IAAI;QACd,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,UAAU,GAAG,CAAC;AAChD,CAAC;AAED,SAAS,MAAM,CAAC,KAAa,EAAE,IAAY;IACzC,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;AACrD,CAAC;AAKD,+DAA+D;AAC/D,MAAM,UAAU,YAAY,CAAC,MAAwB;IACnD,OAAO,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,CAAC;AAC7D,CAAC;AAED,kCAAkC;AAClC,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,MAAwB,EACxB,OAAmB,EACnB,KAAsB,EACtB,MAAoB;IAEpB,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;YACzD,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;gBACzB,OAAO,MAAM,CAAC,KAAK,KAAK,cAAc;oBACpC,CAAC,CAAC;wBACE,MAAM,EAAE,IAAI;wBACZ,WAAW,EAAE,GAAG,OAAO,CAAC,OAAO,6BAA6B,MAAM,CAAC,OAAO,GAAG;qBAC9E;oBACH,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACtD,CAAC;YACD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC3D,MAAM,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;qBACrF,MAAM,CAAC;gBACV,OAAO;oBACL,MAAM,EAAE,KAAK,KAAK,MAAM;oBACxB,WAAW,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,KAAK,KAAK,MAAM,SAAS;iBACjH,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC;YAC7C,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO;oBACL,MAAM,EAAE,QAAQ;oBAChB,WAAW,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,aAAa,KAAK,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG;iBAC9H,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,MAAM,EAAE,CAAC,QAAQ;gBACjB,WAAW,EAAE,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,iBAAiB,KAAK,EAAE;aACxE,CAAC;QACJ,CAAC;QACD,KAAK,WAAW,CAAC,CAAC,CAAC;YACjB,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CACpC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,CAChF,CAAC,MAAM,CAAC;YACT,MAAM,MAAM,GAAG,KAAK,IAAI,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;YACxF,MAAM,MAAM,GACV,MAAM,CAAC,GAAG,KAAK,SAAS;gBACtB,CAAC,CAAC,YAAY,MAAM,CAAC,GAAG,EAAE;gBAC1B,CAAC,CAAC,MAAM,CAAC,GAAG,KAAK,MAAM,CAAC,GAAG;oBACzB,CAAC,CAAC,WAAW,MAAM,CAAC,GAAG,EAAE;oBACzB,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,GAAG,EAAE,CAAC;YACzC,MAAM,MAAM,GACV,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,yBAAyB,MAAM,CAAC,UAAU,GAAG,CAAC;YACvF,OAAO;gBACL,MAAM;gBACN,WAAW,EAAE,GAAG,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,MAAM,KAAK,MAAM,SAAS;aACzF,CAAC;QACJ,CAAC;QACD,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,CAAC,CAAC;gBACf,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;YACxF,IAAI,KAAK,KAAK,CAAC,CAAC;gBACd,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;YACvF,OAAO;gBACL,MAAM,EAAE,MAAM,GAAG,KAAK;gBACtB,WAAW,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,MAAM,GAAG,CAAC,KAAK,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,KAAK,GAAG,CAAC,EAAE;aAClI,CAAC;QACJ,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YACvF,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC;oBACxB,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE;oBAC7E,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,0CAA0C,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9F,CAAC;YACD,OAAO,QAAQ,CAAC,MAAM,KAAK,CAAC;gBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,0CAA0C,MAAM,CAAC,IAAI,EAAE,EAAE;gBACxF,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;QAChG,CAAC;QACD,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YACvD,IAAI,SAAS,IAAI,OAAO;gBAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACjF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,WAAW,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,qFAAqF;iBAC/H,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAC5B;gBACE,IAAI,EAAE,KAAK;gBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,EACD,MAAM,CACP,CAAC;YACF,OAAO;gBACL,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;QACJ,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;YACxD,MAAM,SAAS,GAAG,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC9D,IAAI,OAAO,IAAI,SAAS,EAAE,CAAC;gBACzB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;YACtF,CAAC;YACD,MAAM,GAAG,GAAG,kBAAkB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAC5B;gBACE,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC;gBACxC,GAAG;gBACH,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,EACD,MAAM,CACP,CAAC;YACF,OAAO;gBACL,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,QAAQ,EAAE,GAAG;gBACb,OAAO,EAAE,MAAM,CAAC,OAAO;aACxB,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,KAAK,CAAC,KAAgC;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,WAAW,CAAC;IACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,QAAQ,EAAE,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC;IACtF,OAAO;QACL,MAAM;QACN,WAAW,EAAE,GAAG,IAAI,OAAO,KAAK,CAAC,MAAM,cAAc,MAAM,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;KACnF,CAAC;AACJ,CAAC"}

package/dist/plugin-evals/index.d.ts

@@ -0,0 +1,4 @@
+export { experimental_runPluginEvals } from './runner.js';
+export { PluginEvalError } from './suite.js';
+export type { PluginEvalAblation, PluginEvalAbort, PluginEvalArm, PluginEvalBaselineGrader, PluginEvalCaseAggregates, PluginEvalCaseResult, PluginEvalExpectation, PluginEvalExpectType, PluginEvalFileExistsGrader, PluginEvalGrader, PluginEvalGraderResult, PluginEvalLlmGrader, PluginEvalMockCall, PluginEvalOptions, PluginEvalOutcome, PluginEvalPartialReason, PluginEvalProgressEvent, PluginEvalRegexGrader, PluginEvalReport, PluginEvalResult, PluginEvalRunResult, PluginEvalTarget, PluginEvalToolCall, PluginEvalToolMatcher, PluginEvalToolOrderGrader, PluginEvalToolUsedGrader, PluginEvalVote, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/plugin-evals/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAC1C,YAAY,EACV,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,wBAAwB,EACxB,wBAAwB,EACxB,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,EACtB,mBAAmB,EACnB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,qBAAqB,EACrB,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,EAClB,qBAAqB,EACrB,yBAAyB,EACzB,wBAAwB,EACxB,cAAc,GACf,MAAM,SAAS,CAAC"}

package/dist/plugin-evals/index.js

@@ -0,0 +1,3 @@
+export { experimental_runPluginEvals } from './runner.js';
+export { PluginEvalError } from './suite.js';
+//# sourceMappingURL=index.js.map

package/dist/plugin-evals/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/plugin-evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC"}

package/dist/plugin-evals/judge.d.ts

@@ -0,0 +1,33 @@
+import { type JudgeCogitator } from '../schema.js';
+import type { PluginEvalVote } from './types.js';
+/** Votes per `llm` or `baseline` grader: the grader passes on a majority of PASS votes */
+export declare const JUDGE_VOTES = 3;
+/** What a judge grader asks */
+export type JudgeQuestion = {
+    kind: 'llm';
+    criteria: string;
+    evidence: string;
+    prompt: string;
+} | {
+    kind: 'baseline';
+    criteria: string;
+    reference: string;
+    run: string;
+    prompt: string;
+};
+/** The judge's answer: every vote and what the votes cost */
+export interface JudgeAnswer {
+    votes: PluginEvalVote[];
+    costUsd: number;
+}
+/** Asks the judge model, three votes per question */
+export interface PluginEvalJudge {
+    /** Asks `question`, its judge runs aborted with `signal` */
+    ask(question: JudgeQuestion, signal?: AbortSignal): Promise<JudgeAnswer>;
+}
+/**
+ * A judge on `model`, run by `cogitator`: each question is asked three times as a judge metric of
+ * this package, at temperature 1 so the votes are independent samples
+ */
+export declare function createPluginEvalJudge(cogitator: JudgeCogitator, model: string): PluginEvalJudge;
+//# sourceMappingURL=judge.d.ts.map

package/dist/plugin-evals/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/judge.ts"],"names":[],"mappings":"AAQA,OAAO,EAAqB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAE9C,0FAA0F;AAC1F,eAAO,MAAM,WAAW,IAAI,CAAC;AAE7B,+BAA+B;AAC/B,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACnE;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE3F,6DAA6D;AAC7D,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CAC1E;AAkDD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,GAAG,eAAe,CAqC/F"}

package/dist/plugin-evals/judge.js

@@ -0,0 +1,73 @@
+import { bindJudgeContext, createJudgeMetric, judgeContextFor, } from '../metrics/llm-judge.js';
+import { tagged } from '../metrics/judge-prompt.js';
+import { JudgeConfigSchema } from '../schema.js';
+/** Votes per `llm` or `baseline` grader: the grader passes on a majority of PASS votes */
+export const JUDGE_VOTES = 3;
+const LLM_INSTRUCTIONS = `You grade one run of an AI agent against a rubric. Read the <rubric> and the <evidence> from the run (and the <task> the agent was given), then decide whether the evidence meets the rubric.
+
+Respond with JSON: {"score": 1 for PASS or 0 for FAIL, "reasoning": "<one or two sentences>"}`;
+const BASELINE_INSTRUCTIONS = `You compare one run of an AI agent with a reference run of the same <task>. Read the <criteria>, the <reference> transcript and the <run> transcript, then decide whether the run satisfies the criteria at least as well as the reference does.
+
+Respond with JSON: {"score": 1 for PASS (at least as well) or 0 for FAIL (worse), "reasoning": "<one or two sentences>"}`;
+function verdictScore(verdict) {
+    if (verdict.score === 1 || verdict.score === 0)
+        return { score: verdict.score };
+    return { error: `the judge gave ${verdict.score}, which is neither 1 (PASS) nor 0 (FAIL)` };
+}
+function messageOf(question) {
+    const task = tagged('task', question.prompt);
+    if (question.kind === 'llm') {
+        return `${task}\n${tagged('rubric', question.criteria)}\n${tagged('evidence', question.evidence)}`;
+    }
+    return `${task}\n${tagged('criteria', question.criteria)}\n${tagged('reference', question.reference)}\n${tagged('run', question.run)}`;
+}
+function votesOf(metadata, fallback) {
+    const samples = metadata?.samples;
+    if (!Array.isArray(samples))
+        return Array.from({ length: JUDGE_VOTES }, () => ({ verdict: 'ERROR', reasoning: fallback }));
+    return samples.map((sample) => {
+        if (typeof sample.score !== 'number') {
+            return {
+                verdict: 'ERROR',
+                reasoning: typeof sample.error === 'string' ? sample.error : fallback,
+            };
+        }
+        return {
+            verdict: sample.score === 1 ? 'PASS' : 'FAIL',
+            ...(typeof sample.reasoning === 'string' && { reasoning: sample.reasoning }),
+        };
+    });
+}
+/**
+ * A judge on `model`, run by `cogitator`: each question is asked three times as a judge metric of
+ * this package, at temperature 1 so the votes are independent samples
+ */
+export function createPluginEvalJudge(cogitator, model) {
+    const context = judgeContextFor(cogitator, JudgeConfigSchema.parse({ model, temperature: 1, samples: JUDGE_VOTES }));
+    return {
+        async ask(question, signal) {
+            const message = messageOf(question);
+            const metric = createJudgeMetric(question.kind === 'llm' ? 'llm' : 'baseline', question.kind === 'llm' ? LLM_INSTRUCTIONS : BASELINE_INSTRUCTIONS, { message: () => ({ message }), score: verdictScore });
+            signal?.throwIfAborted();
+            const scoped = signal
+                ? {
+                    ...context,
+                    cogitator: {
+                        run: (options) => context.cogitator.run({ ...options, signal }),
+                    },
+                }
+                : context;
+            const scored = await bindJudgeContext(metric, scoped)({
+                case: { input: question.prompt },
+                output: '',
+                duration: 0,
+            });
+            signal?.throwIfAborted();
+            return {
+                votes: votesOf(scored.metadata, scored.error ?? scored.details ?? 'no verdict'),
+                costUsd: scored.usage?.cost ?? 0,
+            };
+        },
+    };
+}
+//# sourceMappingURL=judge.js.map

package/dist/plugin-evals/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/plugin-evals/judge.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,GAGhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAuB,MAAM,WAAW,CAAC;AAGnE,0FAA0F;AAC1F,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC;AAmB7B,MAAM,gBAAgB,GAAG;;8FAEqE,CAAC;AAE/F,MAAM,qBAAqB,GAAG;;yHAE2F,CAAC;AAE1H,SAAS,YAAY,CAAC,OAAqB;IACzC,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IAChF,OAAO,EAAE,KAAK,EAAE,kBAAkB,OAAO,CAAC,KAAK,0CAA0C,EAAE,CAAC;AAC9F,CAAC;AAED,SAAS,SAAS,CAAC,QAAuB;IACxC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,QAAQ,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC5B,OAAO,GAAG,IAAI,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IACrG,CAAC;IACD,OAAO,GAAG,IAAI,KAAK,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;AACzI,CAAC;AAQD,SAAS,OAAO,CACd,QAA6C,EAC7C,QAAgB;IAEhB,MAAM,OAAO,GAAG,QAAQ,EAAE,OAAO,CAAC;IAClC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QACzB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAChG,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAoB,EAAE,EAAE;QAC1C,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,OAAO;gBACL,OAAO,EAAE,OAAO;gBAChB,SAAS,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ;aACtE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC7C,GAAG,CAAC,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,SAAyB,EAAE,KAAa;IAC5E,MAAM,OAAO,GAAG,eAAe,CAC7B,SAAS,EACT,iBAAiB,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CACzE,CAAC;IACF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM;YACxB,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;YACpC,MAAM,MAAM,GAAG,iBAAiB,CAC9B,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,EAC5C,QAAQ,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,qBAAqB,EAClE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CACtD,CAAC;YACF,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,MAAM;gBACnB,CAAC,CAAC;oBACE,GAAG,OAAO;oBACV,SAAS,EAAE;wBACT,GAAG,EAAE,CAAC,OAAwB,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC;qBACjF;iBACF;gBACH,CAAC,CAAC,OAAO,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,gBAAgB,CACnC,MAAM,EACN,MAAM,CACP,CAAC;gBACA,IAAI,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE;gBAChC,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC;YACH,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,OAAO;gBACL,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,IAAI,YAAY,CAAC;gBAC/E,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,CAAC;aACjC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/plugin-evals/mocks.d.ts

@@ -0,0 +1,34 @@
+import type { PluginEvalAbort, PluginEvalMockTool } from './types.js';
+/** Longest string value an `expect:` pattern checks: a longer one counts as a violation */
+export declare const EXPECT_MAX_LENGTH = 4096;
+/** The mocks of a case, and why the servers that cannot be used cannot, by server */
+export interface CaseMocks {
+    mocks: Record<string, PluginEvalMockTool[]>;
+    problems: Record<string, string>;
+}
+/**
+ * The mocked tools of a case: the suite's mocks with the case's own over them, file by file.
+ * A file of the case's own `mocks/` that cannot be used throws with the file and the reason, since
+ * the case asked for it. A suite file that cannot be used is reported in `problems` under its
+ * server, so it fails only the cases that use that server
+ */
+export declare function loadMocks(suiteDir: string | undefined, caseDir: string): CaseMocks;
+/**
+ * Why `source` is outside the pattern dialect of `expect:` (literals, `.`, escapes, classes, the
+ * quantifiers `* + ? {m,n}` on one of those, an optional `^` and `$`), undefined when it is in it
+ */
+export declare function dialectProblem(source: string): string | undefined;
+/** Why `input` violates the mock's `expect:` checks, undefined when it passes them */
+export declare function expectViolation(mock: PluginEvalMockTool, input: Record<string, unknown>): string | undefined;
+/**
+ * The mock's answer to `input`: its body with `{{input.<field>}}` replaced by the field and
+ * `{{file:<path>}}` by the contents of a file of the server's mock directory, where `<path>` may
+ * hold `{input.<field>}`
+ */
+export declare function renderMock(mock: PluginEvalMockTool, input: Record<string, unknown>): string;
+/** A mock stopping its run: the run scores 0 and reports `aborted` */
+export declare class MockAbortError extends Error {
+    readonly abort: PluginEvalAbort;
+    constructor(abort: PluginEvalAbort);
+}
+//# sourceMappingURL=mocks.d.ts.map

package/dist/plugin-evals/mocks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mocks.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/mocks.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,eAAe,EAGf,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAEjB,2FAA2F;AAC3F,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAiBtC,qFAAqF;AACrF,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC5C,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,SAAS,CA6BlF;AAyID;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAmCjE;AA8BD,sFAAsF;AACtF,wBAAgB,eAAe,CAC7B,IAAI,EAAE,kBAAkB,EACxB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC7B,MAAM,GAAG,SAAS,CAQpB;AAuDD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CA4B3F;AAED,sEAAsE;AACtE,qBAAa,cAAe,SAAQ,KAAK;IAC3B,QAAQ,CAAC,KAAK,EAAE,eAAe;gBAAtB,KAAK,EAAE,eAAe;CAI5C"}