@cogitator-ai/evals 0.13.2 → 0.14.0

package/dist/plugin-evals/mocks.js

@@ -0,0 +1,373 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { errorText, isDirectory, isInside } from './files.js';
+import { isRecord, issuesText, MockFrontmatterSchema, readMarkdown, ToolsListSchema, } from './schema.js';
+/** Longest string value an `expect:` pattern checks: a longer one counts as a violation */
+export const EXPECT_MAX_LENGTH = 4096;
+const EXPECT_TYPES = new Set([
+    'string',
+    'number',
+    'boolean',
+    'array',
+    'object',
+    'null',
+]);
+/**
+ * The mocked tools of a case: the suite's mocks with the case's own over them, file by file.
+ * A file of the case's own `mocks/` that cannot be used throws with the file and the reason, since
+ * the case asked for it. A suite file that cannot be used is reported in `problems` under its
+ * server, so it fails only the cases that use that server
+ */
+export function loadMocks(suiteDir, caseDir) {
+    const merged = new Map();
+    const own = join(caseDir, 'mocks');
+    for (const dir of [suiteDir, own]) {
+        if (dir === undefined || !isDirectory(dir))
+            continue;
+        for (const [server, found] of readMockDir(dir)) {
+            if (dir === own) {
+                const [problem] = found.problems.values();
+                if (problem !== undefined)
+                    throw new Error(problem);
+            }
+            const into = merged.get(server) ?? { tools: new Map(), problems: new Map() };
+            for (const [name, tool] of found.tools) {
+                into.tools.set(name, tool);
+                into.problems.delete(`${name}.md`);
+            }
+            for (const [file, problem] of found.problems)
+                into.problems.set(file, problem);
+            merged.set(server, into);
+        }
+    }
+    const mocks = {};
+    const problems = {};
+    for (const [server, found] of merged) {
+        if (found.problems.size > 0) {
+            problems[server] = [...found.problems.values()].join(', ');
+            continue;
+        }
+        mocks[server] = [...found.tools.values()].sort((a, b) => a.tool.localeCompare(b.tool));
+    }
+    return { mocks, problems };
+}
+function readMockDir(dir) {
+    const servers = new Map();
+    for (const server of readdirSync(dir).sort()) {
+        const serverDir = join(dir, server);
+        if (server.startsWith('.') || !isDirectory(serverDir))
+            continue;
+        const found = { tools: new Map(), problems: new Map() };
+        servers.set(server, found);
+        let schemas = new Map();
+        try {
+            schemas = readToolsList(serverDir);
+        }
+        catch (error) {
+            found.problems.set('_tools.json', errorText(error));
+        }
+        for (const file of readdirSync(serverDir).sort()) {
+            const path = join(serverDir, file);
+            if (file === '_server.md') {
+                found.problems.set(file, `${path}: _server.md agent mocks are not supported by Cogitator's plugin evals, give each tool a fixed mock file`);
+                continue;
+            }
+            if (!file.endsWith('.md') || file.startsWith('_') || file.startsWith('.'))
+                continue;
+            if (!statSync(path).isFile())
+                continue;
+            const tool = file.slice(0, -'.md'.length);
+            try {
+                found.tools.set(tool, readMockFile(path, server, tool, serverDir, schemas.get(tool)));
+            }
+            catch (error) {
+                found.problems.set(file, errorText(error));
+            }
+        }
+    }
+    return servers;
+}
+function readToolsList(serverDir) {
+    const path = join(serverDir, '_tools.json');
+    const found = new Map();
+    if (!existsSync(path))
+        return found;
+    let json;
+    try {
+        json = JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch (error) {
+        throw new Error(`${path}: not valid JSON (${errorText(error)})`, { cause: error });
+    }
+    const parsed = ToolsListSchema.safeParse(json);
+    if (!parsed.success)
+        throw new Error(`${path}: ${issuesText(parsed.error)}`);
+    const tools = Array.isArray(parsed.data) ? parsed.data : parsed.data.tools;
+    for (const tool of tools) {
+        const description = typeof tool.description === 'string' ? tool.description : undefined;
+        const inputSchema = isRecord(tool.inputSchema) ? tool.inputSchema : undefined;
+        found.set(tool.name, {
+            ...(description !== undefined && { description }),
+            ...(inputSchema && { inputSchema }),
+        });
+    }
+    return found;
+}
+function readMockFile(path, server, tool, dir, schema) {
+    const parts = readMarkdown(readFileSync(path, 'utf8'));
+    if ('error' in parts)
+        throw new Error(`${path}: ${parts.error}`);
+    const parsed = MockFrontmatterSchema.safeParse(parts.fields);
+    if (!parsed.success)
+        throw new Error(`${path}: ${issuesText(parsed.error)}`);
+    if (parsed.data.type === 'agent' || parsed.data.abort_when !== undefined) {
+        throw new Error(`${path}: type: agent mocks are not supported by Cogitator's plugin evals, use a fixed mock`);
+    }
+    const expect = {};
+    for (const [field, spec] of Object.entries(parsed.data.expect ?? {})) {
+        const expectation = expectationOf(spec);
+        if ('error' in expectation)
+            throw new Error(`${path}: expect.${field} ${expectation.error}`);
+        expect[field] = expectation;
+    }
+    return {
+        server,
+        tool,
+        body: parts.body.replace(/^\n+/, '').replace(/\s+$/, ''),
+        error: parsed.data.error,
+        expect,
+        dir,
+        ...(schema?.description !== undefined && { description: schema.description }),
+        ...(schema?.inputSchema && { inputSchema: schema.inputSchema }),
+    };
+}
+function isLiteral(value) {
+    return (value === null ||
+        typeof value === 'string' ||
+        typeof value === 'boolean' ||
+        (typeof value === 'number' && Number.isFinite(value)));
+}
+function expectationOf(spec) {
+    if (Array.isArray(spec)) {
+        if (spec.length === 0 || !spec.every(isLiteral)) {
+            return { error: 'must list literal values (strings, numbers, booleans, null)' };
+        }
+        return { kind: 'oneOf', values: spec };
+    }
+    if (typeof spec === 'string') {
+        if (EXPECT_TYPES.has(spec))
+            return { kind: 'type', type: spec };
+        const pattern = /^\/(.*)\/([a-z]*)$/s.exec(spec);
+        if (pattern) {
+            const [, source = '', flags = ''] = pattern;
+            const problem = dialectProblem(source) ?? flagsProblem(flags);
+            if (problem)
+                return { error: `has the pattern ${spec}, which ${problem}` };
+            return { kind: 'pattern', source, flags };
+        }
+        return { kind: 'literal', value: spec };
+    }
+    if (isLiteral(spec))
+        return { kind: 'literal', value: spec };
+    return { error: 'must be a type name, a /regex/, a literal or a list of literals' };
+}
+function flagsProblem(flags) {
+    return /^[is]*$/.test(flags) ? undefined : 'uses a flag other than i and s';
+}
+/**
+ * Why `source` is outside the pattern dialect of `expect:` (literals, `.`, escapes, classes, the
+ * quantifiers `* + ? {m,n}` on one of those, an optional `^` and `$`), undefined when it is in it
+ */
+export function dialectProblem(source) {
+    let at = source.startsWith('^') ? 1 : 0;
+    let end = source.length;
+    if (end > at && source.endsWith('$') && !escapedAt(source, end - 1))
+        end--;
+    while (at < end) {
+        const char = source[at];
+        if (char === '(' || char === ')')
+            return 'has a group, which the dialect does not allow';
+        if (char === '|')
+            return 'has an alternation, write a list of literals instead';
+        if (char === '^' || char === '$')
+            return 'has an anchor that is not at its start or end';
+        if (char === '*' || char === '+' || char === '?' || char === '{') {
+            return 'has a quantifier with nothing to repeat';
+        }
+        if (char === '\\') {
+            const next = source[at + 1];
+            if (next === undefined || at + 1 >= end)
+                return 'ends with a lone backslash';
+            if (/[1-9k]/.test(next))
+                return 'has a backreference, which the dialect does not allow';
+            at += 2;
+        }
+        else if (char === '[') {
+            const close = classEnd(source, at, end);
+            if (close === -1)
+                return 'has an unclosed character class';
+            at = close + 1;
+        }
+        else
+            at++;
+        const quantified = quantifierEnd(source, at, end);
+        if (quantified === -1)
+            return 'has a malformed {m,n} quantifier';
+        if (quantified !== at && quantified < end && source[quantified] === '?') {
+            return 'has a lazy quantifier, which the dialect does not allow';
+        }
+        at = quantified;
+    }
+    try {
+        new RegExp(source);
+    }
+    catch (error) {
+        return `is not a valid regular expression: ${errorText(error)}`;
+    }
+    return undefined;
+}
+function escapedAt(source, index) {
+    let slashes = 0;
+    for (let at = index - 1; at >= 0 && source[at] === '\\'; at--)
+        slashes++;
+    return slashes % 2 === 1;
+}
+function classEnd(source, open, end) {
+    let at = open + 1;
+    if (source[at] === '^')
+        at++;
+    if (source[at] === ']')
+        at++;
+    while (at < end) {
+        if (source[at] === '\\')
+            at += 2;
+        else if (source[at] === ']')
+            return at;
+        else
+            at++;
+    }
+    return -1;
+}
+/** Where a quantifier after an atom ends (`at` itself without one), -1 for a broken `{m,n}` */
+function quantifierEnd(source, at, end) {
+    const char = source[at];
+    if (at >= end)
+        return at;
+    if (char === '*' || char === '+' || char === '?')
+        return at + 1;
+    if (char !== '{')
+        return at;
+    const match = /^\{\d+(?:,\d*)?\}/.exec(source.slice(at, end));
+    return match ? at + match[0].length : -1;
+}
+/** Why `input` violates the mock's `expect:` checks, undefined when it passes them */
+export function expectViolation(mock, input) {
+    for (const [field, expectation] of Object.entries(mock.expect)) {
+        const found = valueAt(input, field);
+        if (!found.present)
+            return `${field} is missing`;
+        const problem = violation(expectation, found.value);
+        if (problem)
+            return `${field} ${problem}`;
+    }
+    return undefined;
+}
+function violation(expectation, value) {
+    switch (expectation.kind) {
+        case 'type':
+            return typeOf(value) === expectation.type ? undefined : `is not a ${expectation.type}`;
+        case 'pattern': {
+            if (typeof value !== 'string')
+                return 'is not a string';
+            if (value.length > EXPECT_MAX_LENGTH) {
+                return `is longer than ${EXPECT_MAX_LENGTH} characters`;
+            }
+            return new RegExp(expectation.source, expectation.flags).test(value)
+                ? undefined
+                : `does not match /${expectation.source}/${expectation.flags}`;
+        }
+        case 'literal':
+            return value === expectation.value
+                ? undefined
+                : `is not ${JSON.stringify(expectation.value)}`;
+        case 'oneOf':
+            return isLiteral(value) && expectation.values.includes(value)
+                ? undefined
+                : `is not one of ${expectation.values.map((v) => JSON.stringify(v)).join(', ')}`;
+    }
+}
+function typeOf(value) {
+    if (value === null)
+        return 'null';
+    if (Array.isArray(value))
+        return 'array';
+    if (typeof value === 'number')
+        return Number.isFinite(value) ? 'number' : undefined;
+    if (typeof value === 'string' || typeof value === 'boolean' || typeof value === 'object') {
+        return typeof value;
+    }
+    return undefined;
+}
+function valueAt(input, path) {
+    let current = input;
+    for (const key of path.split('.')) {
+        if (Array.isArray(current) && /^\d+$/.test(key)) {
+            const index = Number(key);
+            if (index >= current.length)
+                return { present: false };
+            current = current[index];
+        }
+        else if (isRecord(current) && Object.hasOwn(current, key)) {
+            current = current[key];
+        }
+        else
+            return { present: false };
+    }
+    return { present: true, value: current };
+}
+function textOf(value) {
+    if (value === undefined)
+        return '';
+    return typeof value === 'string' ? value : JSON.stringify(value);
+}
+/**
+ * The mock's answer to `input`: its body with `{{input.<field>}}` replaced by the field and
+ * `{{file:<path>}}` by the contents of a file of the server's mock directory, where `<path>` may
+ * hold `{input.<field>}`
+ */
+export function renderMock(mock, input) {
+    return mock.body.replace(/\{\{\s*([^{}]*(?:\{[^{}]*\}[^{}]*)*)\s*\}\}/g, (whole, inner) => {
+        const expression = inner.trim();
+        if (expression.startsWith('input.')) {
+            const found = valueAt(input, expression.slice('input.'.length));
+            return found.present ? textOf(found.value) : '';
+        }
+        if (expression.startsWith('file:')) {
+            const name = expression
+                .slice('file:'.length)
+                .replace(/\{\s*input\.([^{}\s]+)\s*\}/g, (_match, field) => {
+                const found = valueAt(input, field);
+                return found.present ? textOf(found.value) : '';
+            });
+            const path = resolve(mock.dir, name);
+            if (path === mock.dir || !isInside(mock.dir, path)) {
+                throw new Error(`The mock fixture ${name} is outside the mock directory`);
+            }
+            if (!existsSync(path) || !statSync(path).isFile()) {
+                throw new Error(`The mock fixture ${name} does not exist`);
+            }
+            return readFileSync(path, 'utf8');
+        }
+        return whole;
+    });
+}
+/** A mock stopping its run: the run scores 0 and reports `aborted` */
+export class MockAbortError extends Error {
+    abort;
+    constructor(abort) {
+        super(`${abort.server}/${abort.tool}: ${abort.reason}`);
+        this.abort = abort;
+        this.name = 'MockAbortError';
+    }
+}
+//# sourceMappingURL=mocks.js.map

package/dist/plugin-evals/mocks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mocks.js","sourceRoot":"","sources":["../../src/plugin-evals/mocks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,EACL,QAAQ,EACR,UAAU,EACV,qBAAqB,EACrB,YAAY,EACZ,eAAe,GAChB,MAAM,UAAU,CAAC;AAQlB,2FAA2F;AAC3F,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC,MAAM,YAAY,GAAwB,IAAI,GAAG,CAAC;IAChD,QAAQ;IACR,QAAQ;IACR,SAAS;IACT,OAAO;IACP,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AAcH;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,QAA4B,EAAE,OAAe;IACrE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC;YAAE,SAAS;QACrD,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;gBAChB,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC1C,IAAI,OAAO,KAAK,SAAS;oBAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;YAC7E,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACvC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;YACrC,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ;gBAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC/E,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAyC,EAAE,CAAC;IACvD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3D,SAAS;QACX,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAAC,GAAW;IAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACpC,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC;YAAE,SAAS;QAChE,MAAM,KAAK,GAAgB,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC3B,IAAI,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAC;QACjD,IAAI,CAAC;YACH,OAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACjD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnC,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1B,KAAK,CAAC,QAAQ,CAAC,GAAG,CAChB,IAAI,EACJ,GAAG,IAAI,0GAA0G,CAClH,CAAC;gBACF,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YACpF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;gBAAE,SAAS;YACvC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC;gBACH,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD,SAAS,aAAa,CAAC,SAAiB;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2B,CAAC;IACjD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,qBAAqB,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;IAC3E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,WAAW,GAAG,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACxF,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9E,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;YACnB,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,CAAC;YACjD,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,IAAY,EACZ,MAAc,EACd,IAAY,EACZ,GAAW,EACX,MAAmC;IAEnC,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACvD,IAAI,OAAO,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7E,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CACb,GAAG,IAAI,qFAAqF,CAC7F,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAA0C,EAAE,CAAC;IACzD,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QACrE,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,OAAO,IAAI,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,YAAY,KAAK,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC;IAC9B,CAAC;IACD,OAAO;QACL,MAAM;QACN,IAAI;QACJ,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACxD,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;QACxB,MAAM;QACN,GAAG;QACH,GAAG,CAAC,MAAM,EAAE,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7E,GAAG,CAAC,MAAM,EAAE,WAAW,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC;AAID,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,CACL,KAAK,KAAK,IAAI;QACd,OAAO,KAAK,KAAK,QAAQ;QACzB,OAAO,KAAK,KAAK,SAAS;QAC1B,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CACtD,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,IAAa;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,KAAK,EAAE,6DAA6D,EAAE,CAAC;QAClF,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAA4B,EAAE,CAAC;QACxF,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,EAAE,MAAM,GAAG,EAAE,EAAE,KAAK,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC;YAC5C,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,KAAK,CAAC,CAAC;YAC9D,IAAI,OAAO;gBAAE,OAAO,EAAE,KAAK,EAAE,mBAAmB,IAAI,WAAW,OAAO,EAAE,EAAE,CAAC;YAC3E,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC5C,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC7D,OAAO,EAAE,KAAK,EAAE,iEAAiE,EAAE,CAAC;AACtF,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gCAAgC,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,IAAI,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,IAAI,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,IAAI,GAAG,GAAG,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;QAAE,GAAG,EAAE,CAAC;IAC3E,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;QACxB,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,+CAA+C,CAAC;QACzF,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,sDAAsD,CAAC;QAChF,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;YAAE,OAAO,+CAA+C,CAAC;QACzF,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjE,OAAO,yCAAyC,CAAC;QACnD,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,EAAE,GAAG,CAAC,IAAI,GAAG;gBAAE,OAAO,4BAA4B,CAAC;YAC7E,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,uDAAuD,CAAC;YACxF,EAAE,IAAI,CAAC,CAAC;QACV,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,OAAO,iCAAiC,CAAC;YAC3D,EAAE,GAAG,KAAK,GAAG,CAAC,CAAC;QACjB,CAAC;;YAAM,EAAE,EAAE,CAAC;QACZ,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAClD,IAAI,UAAU,KAAK,CAAC,CAAC;YAAE,OAAO,kCAAkC,CAAC;QACjE,IAAI,UAAU,KAAK,EAAE,IAAI,UAAU,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;YACxE,OAAO,yDAAyD,CAAC;QACnE,CAAC;QACD,EAAE,GAAG,UAAU,CAAC;IAClB,CAAC;IACD,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,sCAAsC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAa;IAC9C,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE;QAAE,OAAO,EAAE,CAAC;IACzE,OAAO,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc,EAAE,IAAY,EAAE,GAAW;IACzD,IAAI,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC;IAClB,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG;QAAE,EAAE,EAAE,CAAC;IAC7B,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG;QAAE,EAAE,EAAE,CAAC;IAC7B,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;YAAE,EAAE,IAAI,CAAC,CAAC;aAC5B,IAAI,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG;YAAE,OAAO,EAAE,CAAC;;YAClC,EAAE,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,+FAA+F;AAC/F,SAAS,aAAa,CAAC,MAAc,EAAE,EAAU,EAAE,GAAW;IAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACxB,IAAI,EAAE,IAAI,GAAG;QAAE,OAAO,EAAE,CAAC;IACzB,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;QAAE,OAAO,EAAE,GAAG,CAAC,CAAC;IAChE,IAAI,IAAI,KAAK,GAAG;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC9D,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,eAAe,CAC7B,IAAwB,EACxB,KAA8B;IAE9B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,OAAO;YAAE,OAAO,GAAG,KAAK,aAAa,CAAC;QACjD,MAAM,OAAO,GAAG,SAAS,CAAC,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACpD,IAAI,OAAO;YAAE,OAAO,GAAG,KAAK,IAAI,OAAO,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,SAAS,CAAC,WAAkC,EAAE,KAAc;IACnE,QAAQ,WAAW,CAAC,IAAI,EAAE,CAAC;QACzB,KAAK,MAAM;YACT,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,WAAW,CAAC,IAAI,EAAE,CAAC;QACzF,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,iBAAiB,CAAC;YACxD,IAAI,KAAK,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACrC,OAAO,kBAAkB,iBAAiB,aAAa,CAAC;YAC1D,CAAC;YACD,OAAO,IAAI,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;gBAClE,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,mBAAmB,WAAW,CAAC,MAAM,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;QACnE,CAAC;QACD,KAAK,SAAS;YACZ,OAAO,KAAK,KAAK,WAAW,CAAC,KAAK;gBAChC,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;QACpD,KAAK,OAAO;YACV,OAAO,SAAS,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;gBAC3D,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,iBAAiB,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACvF,CAAC;AACH,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACpF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACzF,OAAO,OAAO,KAA6B,CAAC;IAC9C,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,IAAY;IAC3C,IAAI,OAAO,GAAY,KAAK,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,KAAK,IAAI,OAAO,CAAC,MAAM;gBAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;YACvD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;aAAM,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC;YAC5D,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;;YAAM,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,KAA8B;IACjF,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CACtB,8CAA8C,EAC9C,CAAC,KAAK,EAAE,KAAa,EAAE,EAAE;QACvB,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,CAAC;QACD,IAAI,UAAU,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,UAAU;iBACpB,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;iBACrB,OAAO,CAAC,8BAA8B,EAAE,CAAC,MAAM,EAAE,KAAa,EAAE,EAAE;gBACjE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBACpC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAClD,CAAC,CAAC,CAAC;YACL,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACrC,IAAI,IAAI,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,gCAAgC,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,iBAAiB,CAAC,CAAC;YAC7D,CAAC;YACD,OAAO,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CACF,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,OAAO,cAAe,SAAQ,KAAK;IAClB;IAArB,YAAqB,KAAsB;QACzC,KAAK,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QADrC,UAAK,GAAL,KAAK,CAAiB;QAEzC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF"}

package/dist/plugin-evals/report.d.ts

@@ -0,0 +1,4 @@
+import type { PluginEvalResult } from './types.js';
+/** The result document as a self-contained HTML page: no scripts, no external requests */
+export declare function pluginEvalHtml(result: PluginEvalResult): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/plugin-evals/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/report.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAIV,gBAAgB,EAEjB,MAAM,SAAS,CAAC;AAoKjB,0FAA0F;AAC1F,wBAAgB,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA8B/D"}

package/dist/plugin-evals/report.js

@@ -0,0 +1,177 @@
+import { escapeHtml, foldable, htmlDocument, htmlTiles } from '../reporters/html-format.js';
+const STYLE = `
+.verdict { font-size: 15px; margin: 12px 0; }
+.banner { border: 1px solid var(--warn); color: var(--warn); border-radius: 8px; padding: 8px 12px; margin: 12px 0; }
+.case { border: 1px solid var(--border); border-left: 4px solid var(--border); border-radius: 8px; padding: 12px 16px; margin: 16px 0; }
+.case.regressed { border-left-color: var(--fail); }
+.case.failed-load { border-left-color: var(--fail); }
+.case h2 { margin: 0 0 6px; display: flex; flex-wrap: wrap; gap: 8px; align-items: baseline; }
+.case h3 { font-size: 13px; color: var(--muted); margin: 14px 0 6px; }
+.bar { position: relative; height: 8px; background: var(--surface); border-radius: 4px; margin: 6px 0 10px; }
+.bar span { position: absolute; left: 0; top: 0; bottom: 0; background: var(--pass); border-radius: 4px; }
+.bar span.low { background: var(--fail); }
+.bar i { position: absolute; top: -3px; bottom: -3px; width: 2px; background: var(--text); }
+.run { border-top: 1px solid var(--border); padding: 6px 0; }
+.run-head { display: flex; flex-wrap: wrap; gap: 10px; font-variant-numeric: tabular-nums; }
+.graders { list-style: none; padding: 0; margin: 4px 0 0; }
+.graders li { margin: 2px 0; }
+.chip { display: inline-block; border-radius: 10px; padding: 0 8px; font-size: 12px; font-weight: 600; color: #fff; }
+.chip.pass { background: var(--pass); }
+.chip.fail { background: var(--fail); }
+.chip.skip { background: var(--muted); }
+.badge { display: inline-block; border: 1px solid var(--border); border-radius: 10px; padding: 0 8px; font-size: 11px; color: var(--muted); }
+.notice { color: var(--warn); margin: 2px 0; }
+`;
+function fixed(value, digits = 2) {
+    return value.toFixed(digits);
+}
+function signed(value, digits = 2) {
+    if (value === undefined)
+        return '-';
+    const text = value.toFixed(digits);
+    return value > 0 ? `+${text}` : text;
+}
+function usd(value) {
+    return `$${value.toFixed(value < 1 ? 4 : 2)}`;
+}
+function scoreText(value) {
+    return value === null || value === undefined ? '-' : fixed(value);
+}
+function definitionText(grader) {
+    switch (grader.type) {
+        case 'regex':
+            return `/${grader.pattern}/${grader.flags} ${grader.match} in ${typeof grader.target === 'string' ? grader.target : `file ${grader.target.path}`}`;
+        case 'tool_used':
+            return `${grader.tool}${grader.inputMatch === undefined ? '' : ` /${grader.inputMatch}/`}, ${grader.min} to ${grader.max ?? 'any'} calls`;
+        case 'tool_order':
+            return `${grader.before.tool} before ${grader.after.tool}`;
+        case 'file_exists':
+            return `${grader.exists ? 'a created file matches' : 'no created file matches'} ${grader.path}`;
+        case 'llm':
+            return `${grader.criteria}\n\nfocus: ${typeof grader.focus === 'string' ? grader.focus : `file ${grader.focus.path}`}`;
+        case 'baseline':
+            return `${grader.criteria}\n\nreference: ${grader.baselineFile}`;
+    }
+}
+function graderItem(result) {
+    const chip = result.skipped
+        ? '<span class="chip skip">skipped</span>'
+        : `<span class="chip ${result.passed ? 'pass' : 'fail'}">${result.passed ? 'pass' : 'fail'}</span>`;
+    const badge = result.scored ? '' : ' <span class="badge">plugin-fired indicator</span>';
+    const votes = result.votes
+        ? `<ul>${result.votes
+            .map((vote) => `<li><b>${escapeHtml(vote.verdict)}</b>${vote.reasoning ? ` ${escapeHtml(vote.reasoning)}` : ''}</li>`)
+            .join('')}</ul>`
+        : '';
+    const evidence = result.evidence !== undefined
+        ? `<details><summary>What the judge saw</summary><pre>${escapeHtml(result.evidence)}</pre></details>`
+        : '';
+    const body = `<pre>${escapeHtml(result.explanation)}</pre>${votes}${evidence}`;
+    const open = !result.passed && !result.skipped ? ' open' : '';
+    return `<li>${chip} <code>${escapeHtml(result.name)}</code> <span class="muted">${escapeHtml(result.type)}, weight ${result.weight}</span>${badge}<details${open}><summary>explanation</summary>${body}</details></li>`;
+}
+function runBlock(run) {
+    const head = [
+        `<b>Run ${run.index}</b>`,
+        `score ${fixed(run.score)}`,
+        usd(run.costUsd),
+        `${run.durationSeconds.toFixed(1)}s`,
+        ...(run.error !== null ? [`<span class="fail">${escapeHtml(run.error)}</span>`] : []),
+        ...(run.aborted
+            ? [
+                `<span class="fail">aborted by ${escapeHtml(`${run.aborted.server}/${run.aborted.tool}`)}: ${escapeHtml(run.aborted.reason)}</span>`,
+            ]
+            : []),
+        ...(run.skippedPaidGraders ? ['<span class="warn">judge graders skipped</span>'] : []),
+        ...(run.tempDir ? [`<code>${escapeHtml(run.tempDir)}</code>`] : []),
+    ].join(' ');
+    const graders = run.graders.length > 0
+        ? `<ul class="graders">${run.graders.map(graderItem).join('')}</ul>`
+        : '<p class="muted">not graded</p>';
+    const reply = run.lastMessage !== ''
+        ? `<details><summary>Final reply</summary>${foldable(run.lastMessage, 200)}</details>`
+        : '';
+    return `<div class="run"><div class="run-head">${head}</div>${graders}${reply}</div>`;
+}
+function caseBlock(result, threshold) {
+    const { aggregates } = result;
+    const regressed = aggregates.delta !== undefined && aggregates.delta < 0;
+    const classes = ['case', regressed ? 'regressed' : '', result.error ? 'failed-load' : '']
+        .filter((name) => name !== '')
+        .join(' ');
+    const score = aggregates.score ?? 0;
+    const status = aggregates.passed
+        ? '<span class="pass">pass</span>'
+        : '<span class="fail">fail</span>';
+    const delta = aggregates.delta === undefined ? '' : `<span>delta ${signed(aggregates.delta)}</span>`;
+    const without = aggregates.withoutScore === undefined
+        ? ''
+        : `<span class="muted">without ${scoreText(aggregates.withoutScore)}</span>`;
+    const bar = `<div class="bar"><span class="${score < threshold ? 'low' : ''}" style="width:${(score * 100).toFixed(1)}%"></span><i style="left:${(threshold * 100).toFixed(1)}%"></i></div>`;
+    const notices = result.notices
+        .map((notice) => `<p class="notice">${escapeHtml(notice)}</p>`)
+        .join('');
+    const error = result.error ? `<p class="fail">${escapeHtml(result.error)}</p>` : '';
+    const withRuns = result.arms.with.map(runBlock).join('');
+    const withoutRuns = result.arms.without.map(runBlock).join('');
+    const graders = result.graders
+        .map((grader) => `<li><code>${escapeHtml(grader.name)}</code> <span class="muted">${escapeHtml(grader.type)}</span><pre>${escapeHtml(definitionText(grader))}</pre></li>`)
+        .join('');
+    return [
+        `<section class="${classes}">`,
+        `<h2>${escapeHtml(result.name)} ${status} <span>score ${scoreText(aggregates.score)}</span>${without}${delta}<span class="muted">${usd(aggregates.costUsd)}</span></h2>`,
+        bar,
+        notices,
+        error,
+        withRuns === '' ? '' : `<h3>${result.singleArm ? 'Runs' : 'With plugin'}</h3>${withRuns}`,
+        withoutRuns === '' ? '' : `<h3>Baseline, without the plugin</h3>${withoutRuns}`,
+        result.prompt === ''
+            ? ''
+            : `<details><summary>Prompt</summary><pre>${escapeHtml(result.prompt)}</pre></details>`,
+        graders === '' ? '' : `<details><summary>Graders</summary><ul>${graders}</ul></details>`,
+        '</section>',
+    ].join('\n');
+}
+function verdictLine(result) {
+    const compared = result.cases.filter((c) => c.aggregates.delta !== undefined);
+    if (compared.length === 0 || result.aggregates.meanDelta === undefined)
+        return '';
+    const improved = compared.filter((c) => (c.aggregates.delta ?? 0) > 0).length;
+    const regressed = compared.filter((c) => (c.aggregates.delta ?? 0) < 0).length;
+    const flat = compared.length - improved - regressed;
+    const points = signed(result.aggregates.meanDelta * 100, 1);
+    return `<p class="verdict">Plugin effect: <b>${points} pts</b> vs baseline, improved ${improved}, flat ${flat}, regressed ${regressed} of ${compared.length} case${compared.length === 1 ? '' : 's'}</p>`;
+}
+/** The result document as a self-contained HTML page: no scripts, no external requests */
+export function pluginEvalHtml(result) {
+    const { aggregates } = result;
+    const tiles = htmlTiles([
+        ['Suite score', fixed(aggregates.overallScore)],
+        ['Ablation delta', signed(aggregates.meanDelta)],
+        ['Baseline score', scoreText(aggregates.baselineScore)],
+        [
+            `Cases at ${fixed(result.config.threshold)} or more`,
+            `${aggregates.casesPassed} / ${aggregates.casesTotal}`,
+        ],
+        ['Perfect runs', `${Math.round(aggregates.perfectRuns * 100)}%`],
+        ['Cost', usd(result.costUsd)],
+        ['Duration', `${Math.round(result.durationSeconds)}s`],
+    ]);
+    const banner = result.partial
+        ? `<p class="banner">Partial run (${escapeHtml(result.partialReason ?? 'unknown')}): leave it out of trend charts</p>`
+        : '';
+    const warnings = result.warnings
+        .map((warning) => `<p class="notice">${escapeHtml(warning)}</p>`)
+        .join('');
+    const body = [
+        `<h1>Plugin evals: ${escapeHtml(result.plugin.name)}</h1>`,
+        `<p class="sub">${escapeHtml(result.evalDir)} &middot; ${escapeHtml(result.startedAt)} &middot; Cogitator ${escapeHtml(result.cogitatorVersion)}${result.config.model ? ` &middot; model ${escapeHtml(result.config.model)}` : ''}</p>`,
+        verdictLine(result),
+        banner,
+        warnings,
+        tiles,
+        ...result.cases.map((c) => caseBlock(c, result.config.threshold)),
+    ].join('\n');
+    return htmlDocument(`Plugin evals: ${result.plugin.name}`, body, STYLE);
+}
+//# sourceMappingURL=report.js.map

package/dist/plugin-evals/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/plugin-evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AASzF,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;CAsBb,CAAC;AAEF,SAAS,KAAK,CAAC,KAAa,EAAE,MAAM,GAAG,CAAC;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,MAAM,CAAC,KAAyB,EAAE,MAAM,GAAG,CAAC;IACnD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IACpC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAChD,CAAC;AAED,SAAS,SAAS,CAAC,KAAgC;IACjD,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,cAAc,CAAC,MAAwB;IAC9C,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO;YACV,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,OAAO,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACrJ,KAAK,WAAW;YACd,OAAO,GAAG,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,UAAU,GAAG,KAAK,MAAM,CAAC,GAAG,OAAO,MAAM,CAAC,GAAG,IAAI,KAAK,QAAQ,CAAC;QAC5I,KAAK,YAAY;YACf,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC7D,KAAK,aAAa;YAChB,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,yBAAyB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAClG,KAAK,KAAK;YACR,OAAO,GAAG,MAAM,CAAC,QAAQ,cAAc,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QACzH,KAAK,UAAU;YACb,OAAO,GAAG,MAAM,CAAC,QAAQ,kBAAkB,MAAM,CAAC,YAAY,EAAE,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAA8B;IAChD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO;QACzB,CAAC,CAAC,wCAAwC;QAC1C,CAAC,CAAC,qBAAqB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC;IACtG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,oDAAoD,CAAC;IACxF,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK;QACxB,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK;aAChB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,UAAU,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CACzG;aACA,IAAI,CAAC,EAAE,CAAC,OAAO;QACpB,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,sDAAsD,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,kBAAkB;QACrG,CAAC,CAAC,EAAE,CAAC;IACT,MAAM,IAAI,GAAG,QAAQ,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,KAAK,GAAG,QAAQ,EAAE,CAAC;IAC/E,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9D,OAAO,OAAO,IAAI,UAAU,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,MAAM,UAAU,KAAK,WAAW,IAAI,kCAAkC,IAAI,iBAAiB,CAAC;AAC1N,CAAC;AAED,SAAS,QAAQ,CAAC,GAAwB;IACxC,MAAM,IAAI,GAAG;QACX,UAAU,GAAG,CAAC,KAAK,MAAM;QACzB,SAAS,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE;QAC3B,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;QAChB,GAAG,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;QACpC,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,GAAG,CAAC,GAAG,CAAC,OAAO;YACb,CAAC,CAAC;gBACE,iCAAiC,UAAU,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS;aACrI;YACH,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,iDAAiD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACZ,MAAM,OAAO,GACX,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;QACpB,CAAC,CAAC,uBAAuB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO;QACpE,CAAC,CAAC,iCAAiC,CAAC;IACxC,MAAM,KAAK,GACT,GAAG,CAAC,WAAW,KAAK,EAAE;QACpB,CAAC,CAAC,0CAA0C,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,YAAY;QACtF,CAAC,CAAC,EAAE,CAAC;IACT,OAAO,0CAA0C,IAAI,SAAS,OAAO,GAAG,KAAK,QAAQ,CAAC;AACxF,CAAC;AAED,SAAS,SAAS,CAAC,MAA4B,EAAE,SAAiB;IAChE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAC9B,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;SACtF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;SAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM;QAC9B,CAAC,CAAC,gCAAgC;QAClC,CAAC,CAAC,gCAAgC,CAAC;IACrC,MAAM,KAAK,GACT,UAAU,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC;IACzF,MAAM,OAAO,GACX,UAAU,CAAC,YAAY,KAAK,SAAS;QACnC,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,+BAA+B,SAAS,CAAC,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC;IACjF,MAAM,GAAG,GAAG,iCAAiC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,kBAAkB,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,4BAA4B,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IAC7L,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;SAC3B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC;SAC9D,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO;SAC3B,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,aAAa,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,aAAa,CAC3J;SACA,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,OAAO;QACL,mBAAmB,OAAO,IAAI;QAC9B,OAAO,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,gBAAgB,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,OAAO,GAAG,KAAK,uBAAuB,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,cAAc;QACxK,GAAG;QACH,OAAO;QACP,KAAK;QACL,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,QAAQ,QAAQ,EAAE;QACzF,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,wCAAwC,WAAW,EAAE;QAC/E,MAAM,CAAC,MAAM,KAAK,EAAE;YAClB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,0CAA0C,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB;QACzF,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,0CAA0C,OAAO,iBAAiB;QACxF,YAAY;KACb,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,MAAwB;IAC3C,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC9E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAClF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9E,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC/E,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;IACpD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5D,OAAO,wCAAwC,MAAM,kCAAkC,QAAQ,UAAU,IAAI,eAAe,SAAS,OAAO,QAAQ,CAAC,MAAM,QAAQ,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;AAC5M,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,cAAc,CAAC,MAAwB;IACrD,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC;IAC9B,MAAM,KAAK,GAAG,SAAS,CAAC;QACtB,CAAC,aAAa,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC/C,CAAC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC,gBAAgB,EAAE,SAAS,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACvD;YACE,YAAY,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU;YACpD,GAAG,UAAU,CAAC,WAAW,MAAM,UAAU,CAAC,UAAU,EAAE;SACvD;QACD,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,WAAW,GAAG,GAAG,CAAC,GAAG,CAAC;QAChE,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC;KACvD,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO;QAC3B,CAAC,CAAC,kCAAkC,UAAU,CAAC,MAAM,CAAC,aAAa,IAAI,SAAS,CAAC,qCAAqC;QACtH,CAAC,CAAC,EAAE,CAAC;IACP,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ;SAC7B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,qBAAqB,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC;SAChE,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,MAAM,IAAI,GAAG;QACX,qBAAqB,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO;QAC1D,kBAAkB,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,uBAAuB,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,mBAAmB,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM;QACvO,WAAW,CAAC,MAAM,CAAC;QACnB,MAAM;QACN,QAAQ;QACR,KAAK;QACL,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;KAClE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,YAAY,CAAC,iBAAiB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;AAC1E,CAAC"}

package/dist/plugin-evals/runner.d.ts

@@ -0,0 +1,23 @@
+import type { PluginEvalOptions, PluginEvalReport } from './types.js';
+/**
+ * Experimental. Runs a plugin's eval suite in the format of Claude Code's `claude plugin eval`:
+ * each case's prompt several times in a fresh workspace with the plugin loaded and, for the
+ * no-plugin baseline, without it, scores every run with the case's graders (`regex`,
+ * `tool_used`, `tool_order`, `file_exists`, and `llm` and `baseline` judged by a majority of three
+ * votes) and writes `aggregate-result.json` and a self-contained `report.html`.
+ *
+ * Needs `@cogitator-ai/core`. Throws `PluginEvalError` when the suite cannot start: no plugin, no
+ * case, an invalid option. A case that fails to load is reported in the result instead.
+ *
+ * @example
+ * ```ts
+ * const { result, reportFile } = await experimental_runPluginEvals({
+ *   target: './my-plugin',
+ *   llm: { defaultModel: 'anthropic/claude-sonnet-5-5' },
+ *   threshold: 0.8,
+ * });
+ * console.log(result.aggregates.overallScore, reportFile);
+ * ```
+ */
+export declare function experimental_runPluginEvals(input: PluginEvalOptions): Promise<PluginEvalReport>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/plugin-evals/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/runner.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAMV,iBAAiB,EAGjB,gBAAgB,EAGjB,MAAM,SAAS,CAAC;AAsLjB;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,2BAA2B,CAC/C,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAwZ3B"}