@cogitator-ai/evals 0.13.2 → 0.14.0

package/dist/plugin-evals/suite.js

@@ -0,0 +1,476 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { basename, dirname, isAbsolute, join, posix, relative, resolve } from 'node:path';
+import { errorText, isDirectory, isFile, isInside, plainRelativePath, posixPath } from './files.js';
+import { globMatches } from './glob.js';
+import { loadMocks } from './mocks.js';
+import { CaseYamlSchema, DEFAULT_MAX_TURNS, DEFAULT_RUNS, DEFAULT_TIMEOUT_SECONDS, GraderSchema, isRecord, issuesText, PromptFrontmatterSchema, readMarkdown, readYaml, } from './schema.js';
+import { parseTranscript } from './transcript.js';
+/** A plugin eval run that cannot start: no plugin, no cases, a bad option */
+export class PluginEvalError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PluginEvalError';
+    }
+}
+const CLAUDE_MANIFEST = join('.claude-plugin', 'plugin.json');
+const COGITATOR_MANIFEST = join('.cogitator-plugin', 'plugin.json');
+const CASE_FILES = ['prompt.md', 'case.yaml'];
+/** Whether `dir` holds a plugin manifest Cogitator reads */
+export function isPluginRoot(dir) {
+    return isFile(join(dir, CLAUDE_MANIFEST)) || isFile(join(dir, COGITATOR_MANIFEST));
+}
+function enclosingPlugin(dir) {
+    let current = dir;
+    for (;;) {
+        if (isPluginRoot(current))
+            return current;
+        const parent = dirname(current);
+        if (parent === current)
+            return undefined;
+        current = parent;
+    }
+}
+/** The `experimental.evals` of the plugin's manifests, Cogitator's over Claude Code's */
+function manifestEvalDir(root) {
+    let found;
+    for (const file of [CLAUDE_MANIFEST, COGITATOR_MANIFEST]) {
+        const path = join(root, file);
+        if (!isFile(path))
+            continue;
+        try {
+            const json = JSON.parse(readFileSync(path, 'utf8'));
+            if (isRecord(json) && isRecord(json.experimental) && json.experimental.evals !== undefined) {
+                found = json.experimental.evals;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return found;
+}
+function evalDirOf(root, flag, warnings) {
+    if (flag !== undefined) {
+        const checked = plainRelativePath(flag);
+        if ('problem' in checked) {
+            throw new PluginEvalError(`--eval-dir ${flag} ${checked.problem}: give a relative path of plain directory names such as qa or quality/evals`);
+        }
+        return checked.path;
+    }
+    const fromManifest = manifestEvalDir(root);
+    if (fromManifest === undefined)
+        return 'evals';
+    const value = Array.isArray(fromManifest) ? fromManifest[0] : fromManifest;
+    const checked = typeof value === 'string'
+        ? plainRelativePath(value)
+        : { problem: Array.isArray(fromManifest) ? 'has no path first' : 'is not a string' };
+    if ('problem' in checked) {
+        warnings.push(`Warning: experimental.evals ${JSON.stringify(fromManifest)} in plugin.json ${checked.problem}, using evals/`);
+        return 'evals';
+    }
+    return checked.path;
+}
+/** Resolves the target, finds the eval directory and loads every case the filters keep */
+export async function loadSuite(options) {
+    const warnings = [];
+    const target = options.target ?? '.';
+    const path = resolve(options.cwd, target);
+    let pluginRoot;
+    let caseDir;
+    let named = false;
+    if (existsSync(path)) {
+        if (isFile(path)) {
+            if (!CASE_FILES.includes(basename(path))) {
+                throw new PluginEvalError(`${target} is not a prompt.md or case.yaml file`);
+            }
+            caseDir = dirname(path);
+            const root = enclosingPlugin(caseDir);
+            if (!root) {
+                throw new PluginEvalError(`No plugin encloses ${target}: no directory above it has ${CLAUDE_MANIFEST} or ${COGITATOR_MANIFEST}`);
+            }
+            pluginRoot = root;
+        }
+        else {
+            if (!isPluginRoot(path)) {
+                throw new PluginEvalError(`${target} is not a plugin directory: it has no ${CLAUDE_MANIFEST} or ${COGITATOR_MANIFEST}`);
+            }
+            pluginRoot = path;
+        }
+    }
+    else if (/^[A-Za-z0-9][\w.-]*(?:@[\w.-]+)?$/.test(target)) {
+        if (target.endsWith('@skills-dir')) {
+            throw new PluginEvalError(`${target}: skills-directory plugins are not supported by Cogitator, pass the plugin directory instead`);
+        }
+        const installed = await options.installed(target);
+        if (!installed) {
+            throw new PluginEvalError(`No plugin directory or case file at ${target}, and no installed plugin is named ${target}`);
+        }
+        pluginRoot = installed;
+        named = true;
+    }
+    else {
+        throw new PluginEvalError(`No plugin directory or case file at ${target}`);
+    }
+    const evalRelative = evalDirOf(pluginRoot, options.evalDir, warnings);
+    const evalDir = join(pluginRoot, evalRelative);
+    const resultsDir = named
+        ? join(resolve(options.cwd), evalRelative, 'results')
+        : join(caseDir && !isInside(evalDir, caseDir) ? dirname(caseDir) : evalDir, 'results');
+    const suiteMocks = join(evalDir, 'mocks');
+    const caseDirs = caseDir ? [caseDir] : findCaseDirs(evalDir);
+    const names = new Set();
+    const cases = [];
+    for (const dir of caseDirs) {
+        const casePath = posixPath(relative(evalDir, dir)) || basename(dir);
+        try {
+            const loaded = loadCase(dir, casePath, pluginRoot, isInside(evalDir, dir) ? suiteMocks : undefined);
+            if (!keep(loaded.name, loaded.tags, options))
+                continue;
+            if (names.has(loaded.name)) {
+                cases.push({
+                    name: loaded.name,
+                    path: casePath,
+                    error: `another case is also named ${loaded.name}`,
+                });
+                continue;
+            }
+            names.add(loaded.name);
+            cases.push({ name: loaded.name, path: casePath, case: loaded });
+        }
+        catch (error) {
+            const loose = looseIdentity(dir);
+            if (!keep(loose.name, loose.tags, options))
+                continue;
+            cases.push({ name: loose.name, path: casePath, error: errorText(error) });
+        }
+    }
+    if (cases.length === 0) {
+        const filtered = options.caseGlob !== undefined || (options.tags?.length ?? 0) > 0;
+        throw new PluginEvalError(filtered
+            ? `No eval cases found: no case under ${evalDir} matches the --case and --tag filters`
+            : `No eval cases found under ${evalDir}: add <case>/prompt.md or <case>/case.yaml`);
+    }
+    return { pluginRoot, evalDir, resultsDir, named, cases, warnings };
+}
+/**
+ * Whether the filters keep a case. `tags` is undefined for a case whose tags could not be read:
+ * a tag filter keeps it, so a broken case still counts and fails the run
+ */
+function keep(name, tags, options) {
+    if (options.caseGlob !== undefined && !globMatches(options.caseGlob, name))
+        return false;
+    const wanted = options.tags ?? [];
+    if (wanted.length === 0 || tags === undefined)
+        return true;
+    return tags.some((tag) => wanted.includes(tag));
+}
+function stringList(value) {
+    return Array.isArray(value) && value.every((item) => typeof item === 'string')
+        ? value
+        : undefined;
+}
+/**
+ * The name and tags of a case that did not load, as far as its files can still be read: the
+ * `prompt.md` frontmatter over `case.yaml`, the directory name without either
+ */
+function looseIdentity(dir) {
+    let name;
+    let rawTags;
+    let readable = true;
+    const yamlPath = join(dir, 'case.yaml');
+    if (isFile(yamlPath)) {
+        const parsed = readYaml(readFileSync(yamlPath, 'utf8'));
+        if ('error' in parsed || !isRecord(parsed.value))
+            readable = false;
+        else {
+            if (typeof parsed.value.name === 'string')
+                name = parsed.value.name;
+            if (parsed.value.tags !== undefined)
+                rawTags = parsed.value.tags;
+        }
+    }
+    const promptPath = join(dir, 'prompt.md');
+    if (isFile(promptPath)) {
+        const parts = readMarkdown(readFileSync(promptPath, 'utf8'));
+        if ('error' in parts)
+            readable = false;
+        else {
+            if (typeof parts.fields.name === 'string')
+                name = parts.fields.name;
+            if (parts.fields.tags !== undefined)
+                rawTags = parts.fields.tags;
+        }
+    }
+    const tags = rawTags !== undefined ? stringList(rawTags) : readable ? [] : undefined;
+    return { name: name?.trim() || basename(dir), ...(tags && { tags }) };
+}
+function isCaseDir(dir) {
+    return CASE_FILES.some((file) => isFile(join(dir, file)));
+}
+/** The case directories under the eval directory, sorted, not looking inside a case */
+function findCaseDirs(evalDir) {
+    if (!isDirectory(evalDir))
+        return [];
+    const found = [];
+    const walk = (dir) => {
+        for (const entry of readdirSync(dir).sort()) {
+            if (entry.startsWith('.') || entry === 'node_modules')
+                continue;
+            if (dir === evalDir && (entry === 'results' || entry === 'mocks'))
+                continue;
+            const full = join(dir, entry);
+            if (!isDirectory(full))
+                continue;
+            if (isCaseDir(full))
+                found.push(full);
+            else
+                walk(full);
+        }
+    };
+    walk(evalDir);
+    return found;
+}
+function readCaseYaml(dir) {
+    const path = join(dir, 'case.yaml');
+    if (!isFile(path))
+        return undefined;
+    const parsed = readYaml(readFileSync(path, 'utf8'));
+    if ('error' in parsed)
+        throw new Error(`invalid case.yaml: ${parsed.error}`);
+    const checked = CaseYamlSchema.safeParse(parsed.value ?? {});
+    if (!checked.success)
+        throw new Error(`invalid case.yaml: ${issuesText(checked.error)}`);
+    return checked.data;
+}
+function readPrompt(dir) {
+    const path = join(dir, 'prompt.md');
+    if (!isFile(path))
+        return undefined;
+    const parts = readMarkdown(readFileSync(path, 'utf8'));
+    if ('error' in parts)
+        throw new Error(`invalid prompt.md: ${parts.error}`);
+    const checked = PromptFrontmatterSchema.safeParse(parts.fields);
+    if (!checked.success)
+        throw new Error(`invalid prompt.md frontmatter: ${issuesText(checked.error)}`);
+    return { fields: checked.data, body: parts.body.trim() };
+}
+/** A file or directory a case names, which must stay inside the case directory */
+function inCase(dir, path, field) {
+    const full = resolve(dir, path);
+    if (isAbsolute(path) || !isInside(dir, full) || full === dir) {
+        throw new Error(`invalid case.yaml: ${field} ${path} must be a path inside the case directory`);
+    }
+    return full;
+}
+/** Loads one case: its prompt, settings, graders, context and mocks */
+export function loadCase(dir, path, pluginRoot, suiteMocks) {
+    const yaml = readCaseYaml(dir);
+    const prompt = readPrompt(dir);
+    const front = prompt?.fields ?? {};
+    const execution = yaml?.execution ?? {};
+    const text = prompt && prompt.body !== '' ? prompt.body : (execution.prompt?.trim() ?? '');
+    if (text === '') {
+        throw new Error('the case has no prompt: write it as the body of prompt.md or as execution.prompt in case.yaml');
+    }
+    const graders = [];
+    for (const [index, entry] of (yaml?.graders ?? []).entries()) {
+        if (typeof entry.name !== 'string' || entry.name.trim() === '') {
+            throw new Error(`invalid case.yaml: graders.${index} needs a name`);
+        }
+        graders.push(graderOf(entry, entry.name.trim(), `graders.${index}`, dir));
+    }
+    const graderDir = join(dir, 'graders');
+    if (isDirectory(graderDir)) {
+        for (const file of readdirSync(graderDir).sort()) {
+            if (!file.endsWith('.md') || file.startsWith('.'))
+                continue;
+            const parts = readMarkdown(readFileSync(join(graderDir, file), 'utf8'));
+            if ('error' in parts)
+                throw new Error(`invalid graders/${file}: ${parts.error}`);
+            const body = parts.body.trim();
+            const fields = { ...parts.fields };
+            if (body !== '') {
+                if ((fields.type === 'llm' || fields.type === 'baseline') &&
+                    fields.criteria === undefined) {
+                    fields.criteria = body;
+                }
+                if (fields.type === 'regex' && fields.pattern === undefined)
+                    fields.pattern = body;
+            }
+            graders.push(graderOf(fields, file.slice(0, -'.md'.length), `graders/${file}`, dir));
+        }
+    }
+    if (graders.length === 0) {
+        throw new Error('invalid case.yaml: graders: a case needs at least one grader, as graders/<name>.md or a graders: entry in case.yaml');
+    }
+    const graderNames = new Set();
+    for (const grader of graders) {
+        if (graderNames.has(grader.name))
+            throw new Error(`two graders are named ${grader.name}`);
+        graderNames.add(grader.name);
+    }
+    const pluginPaths = front.plugins ?? yaml?.plugins;
+    const plugins = pluginPaths
+        ? pluginPaths.map((plugin) => {
+            const full = resolve(dir, plugin);
+            if (!isPluginRoot(full)) {
+                throw new Error(`plugins: ${plugin} is not a plugin directory (from the case directory)`);
+            }
+            return full;
+        })
+        : [pluginRoot];
+    const context = yaml?.context ?? {};
+    const scaffoldScript = context.scaffold_script === undefined
+        ? undefined
+        : inCase(dir, context.scaffold_script, 'context.scaffold_script');
+    if (scaffoldScript && !isFile(scaffoldScript)) {
+        throw new Error(`invalid case.yaml: context.scaffold_script ${context.scaffold_script} does not exist`);
+    }
+    const historyFile = context.history_file === undefined
+        ? undefined
+        : inCase(dir, context.history_file, 'context.history_file');
+    if (historyFile) {
+        if (!isFile(historyFile)) {
+            throw new Error(`invalid case.yaml: context.history_file ${context.history_file} does not exist`);
+        }
+        const parsed = parseTranscript(readFileSync(historyFile, 'utf8'));
+        if ('error' in parsed) {
+            throw new Error(`invalid case.yaml: context.history_file ${context.history_file}: ${parsed.error}`);
+        }
+    }
+    const addDirs = (context.add_dirs ?? []).map((added) => {
+        const checked = plainRelativePath(added);
+        if ('problem' in checked) {
+            throw new Error(`invalid case.yaml: context.add_dirs ${added} ${checked.problem}`);
+        }
+        if (!isDirectory(join(dir, checked.path))) {
+            throw new Error(`invalid case.yaml: context.add_dirs ${added} is not a directory of the case`);
+        }
+        return checked.path;
+    });
+    let mocks;
+    try {
+        mocks = loadMocks(suiteMocks, dir);
+    }
+    catch (error) {
+        throw new Error(`invalid mock: ${errorText(error)}`, { cause: error });
+    }
+    const name = front.name ?? yaml?.name ?? basename(dir);
+    const description = front.description ?? yaml?.description;
+    const expectedOutcome = front.expected_outcome ?? yaml?.expected_outcome;
+    const model = front.model ?? execution.model;
+    const appendSystemPrompt = front.append_system_prompt ?? execution.append_system_prompt;
+    return {
+        name,
+        dir,
+        path,
+        ...(description !== undefined && { description }),
+        ...(expectedOutcome !== undefined && { expectedOutcome }),
+        tags: front.tags ?? yaml?.tags ?? [],
+        plugins,
+        runs: front.runs ?? yaml?.runs ?? DEFAULT_RUNS,
+        ...(model !== undefined && { model }),
+        maxTurns: front.max_turns ?? execution.max_turns ?? DEFAULT_MAX_TURNS,
+        timeoutSeconds: front.timeout_seconds ?? execution.timeout_seconds ?? DEFAULT_TIMEOUT_SECONDS,
+        allowedTools: front.allowed_tools ?? execution.allowed_tools ?? [],
+        ...(appendSystemPrompt !== undefined && { appendSystemPrompt }),
+        env: { ...execution.env, ...front.env },
+        prompt: text,
+        context: {
+            ...(scaffoldScript && { scaffoldScript }),
+            ...(historyFile && { historyFile }),
+            addDirs,
+        },
+        graders,
+        mocks: mocks.mocks,
+        mockProblems: mocks.problems,
+    };
+}
+function matcherOf(value) {
+    if (typeof value === 'string')
+        return { tool: value };
+    return value.input_match === undefined
+        ? { tool: value.tool }
+        : { tool: value.tool, inputMatch: value.input_match };
+}
+function workspacePathProblem(path) {
+    const normalized = posix.normalize(path.replace(/\\/g, '/'));
+    const outside = isAbsolute(path) || normalized === '.' || normalized === '..' || normalized.startsWith('../');
+    return outside ? 'must be a path inside the workspace' : undefined;
+}
+function graderOf(fields, name, where, dir) {
+    const parsed = GraderSchema.safeParse({ ...fields, name });
+    if (!parsed.success)
+        throw new Error(`invalid ${where}: ${issuesText(parsed.error)}`);
+    const grader = parsed.data;
+    const base = {
+        name,
+        weight: grader.weight,
+        ...(grader.arm !== undefined && { arm: grader.arm }),
+    };
+    switch (grader.type) {
+        case 'regex': {
+            const target = grader.target;
+            if (typeof target === 'object') {
+                const problem = workspacePathProblem(target.path);
+                if (problem)
+                    throw new Error(`invalid ${where}: target.path ${problem}`);
+            }
+            return {
+                ...base,
+                type: 'regex',
+                pattern: grader.pattern,
+                flags: grader.flags,
+                match: grader.match,
+                target,
+            };
+        }
+        case 'tool_used':
+            if (grader.max !== undefined && grader.max < grader.min) {
+                throw new Error(`invalid ${where}: max is below min`);
+            }
+            return {
+                ...base,
+                type: 'tool_used',
+                tool: grader.tool,
+                ...(grader.input_match !== undefined && { inputMatch: grader.input_match }),
+                min: grader.min,
+                ...(grader.max !== undefined && { max: grader.max }),
+            };
+        case 'tool_order':
+            return {
+                ...base,
+                type: 'tool_order',
+                before: matcherOf(grader.before),
+                after: matcherOf(grader.after),
+            };
+        case 'file_exists':
+            return { ...base, type: 'file_exists', path: grader.path, exists: grader.exists };
+        case 'llm': {
+            const focus = grader.focus;
+            if (typeof focus === 'object') {
+                const problem = workspacePathProblem(focus.path);
+                if (problem)
+                    throw new Error(`invalid ${where}: focus.path ${problem}`);
+            }
+            return { ...base, type: 'llm', criteria: grader.criteria, focus };
+        }
+        case 'baseline': {
+            const file = inCase(dir, grader.baseline_file, `${where} baseline_file`);
+            if (!file.endsWith('.jsonl') || !isFile(file)) {
+                throw new Error(`invalid ${where}: baseline_file ${grader.baseline_file} must be a .jsonl file of the case directory`);
+            }
+            const parsed = parseTranscript(readFileSync(file, 'utf8'));
+            if ('error' in parsed) {
+                throw new Error(`invalid ${where}: baseline_file ${grader.baseline_file}: ${parsed.error}`);
+            }
+            return {
+                ...base,
+                type: 'baseline',
+                baselineFile: grader.baseline_file,
+                criteria: grader.criteria,
+            };
+        }
+    }
+}
+//# sourceMappingURL=suite.js.map

package/dist/plugin-evals/suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../src/plugin-evals/suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1F,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACjG,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,YAAY,EACZ,uBAAuB,EACvB,YAAY,EACZ,QAAQ,EACR,UAAU,EACV,uBAAuB,EACvB,YAAY,EACZ,QAAQ,GAGT,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQ/C,6EAA6E;AAC7E,MAAM,OAAO,eAAgB,SAAQ,KAAK;IACxC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;AAC9D,MAAM,kBAAkB,GAAG,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;AACpE,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,WAAW,CAAU,CAAC;AAiCvD,4DAA4D;AAC5D,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,SAAS,CAAC;QACR,IAAI,YAAY,CAAC,OAAO,CAAC;YAAE,OAAO,OAAO,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,OAAO;YAAE,OAAO,SAAS,CAAC;QACzC,OAAO,GAAG,MAAM,CAAC;IACnB,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,KAAc,CAAC;IACnB,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC;YACH,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC7D,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3F,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,IAAwB,EAAE,QAAkB;IAC3E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,IAAI,eAAe,CACvB,cAAc,IAAI,IAAI,OAAO,CAAC,OAAO,6EAA6E,CACnH,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC;IACD,MAAM,YAAY,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IAC/C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAE,YAA0B,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;IAC1F,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC;QAC1B,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;IACzF,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;QACzB,QAAQ,CAAC,IAAI,CACX,+BAA+B,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,mBAAmB,OAAO,CAAC,OAAO,gBAAgB,CAC9G,CAAC;QACF,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAqB;IACnD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,GAAG,CAAC;IACrC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC1C,IAAI,UAAkB,CAAC;IACvB,IAAI,OAA2B,CAAC;IAChC,IAAI,KAAK,GAAG,KAAK,CAAC;IAClB,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACjB,IAAI,CAAE,UAAgC,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,eAAe,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;YAC9E,CAAC;YACD,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,eAAe,CACvB,sBAAsB,MAAM,+BAA+B,eAAe,OAAO,kBAAkB,EAAE,CACtG,CAAC;YACJ,CAAC;YACD,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,eAAe,CACvB,GAAG,MAAM,yCAAyC,eAAe,OAAO,kBAAkB,EAAE,CAC7F,CAAC;YACJ,CAAC;YACD,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;SAAM,IAAI,mCAAmC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5D,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,eAAe,CACvB,GAAG,MAAM,8FAA8F,CACxG,CAAC;QACJ,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAClD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,eAAe,CACvB,uCAAuC,MAAM,sCAAsC,MAAM,EAAE,CAC5F,CAAC;QACJ,CAAC;QACD,UAAU,GAAG,SAAS,CAAC;QACvB,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,eAAe,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,YAAY,GAAG,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,KAAK;QACtB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,YAAY,EAAE,SAAS,CAAC;QACrD,CAAC,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACzF,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CACrB,GAAG,EACH,QAAQ,EACR,UAAU,EACV,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAChD,CAAC;YACF,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC;gBAAE,SAAS;YACvD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,8BAA8B,MAAM,CAAC,IAAI,EAAE;iBACnD,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC;gBAAE,SAAS;YACrD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACnF,MAAM,IAAI,eAAe,CACvB,QAAQ;YACN,CAAC,CAAC,sCAAsC,OAAO,uCAAuC;YACtF,CAAC,CAAC,6BAA6B,OAAO,4CAA4C,CACrF,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAmC,EAAE,OAAqB;IACpF,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACzF,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;IAClC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC3D,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC;QAC5E,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,GAAW;IAChC,IAAI,IAAwB,CAAC;IAC7B,IAAI,OAAgB,CAAC;IACrB,IAAI,QAAQ,GAAG,IAAI,CAAC;IACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QACxD,IAAI,OAAO,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,QAAQ,GAAG,KAAK,CAAC;aAC9D,CAAC;YACJ,IAAI,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ;gBAAE,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;YACpE,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QACnE,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7D,IAAI,OAAO,IAAI,KAAK;YAAE,QAAQ,GAAG,KAAK,CAAC;aAClC,CAAC;YACJ,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ;gBAAE,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;YACpE,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS;gBAAE,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;QACnE,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrF,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AACxE,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,uFAAuF;AACvF,SAAS,YAAY,CAAC,OAAe;IACnC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,EAAE;QAC3B,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,cAAc;gBAAE,SAAS;YAChE,IAAI,GAAG,KAAK,OAAO,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,OAAO,CAAC;gBAAE,SAAS;YAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC;gBAAE,SAAS;YACjC,IAAI,SAAS,CAAC,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;gBACjC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,CAAC;IACd,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,IAAI,OAAO,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7E,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7D,IAAI,CAAC,OAAO,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzF,OAAO,OAAO,CAAC,IAAI,CAAC;AACtB,CAAC;AAED,SAAS,UAAU,CAAC,GAAW;IAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IACpC,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACvD,IAAI,OAAO,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,uBAAuB,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAChE,IAAI,CAAC,OAAO,CAAC,OAAO;QAClB,MAAM,IAAI,KAAK,CAAC,kCAAkC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACjF,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;AAC3D,CAAC;AAED,kFAAkF;AAClF,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY,EAAE,KAAa;IACtD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,IAAI,IAAI,2CAA2C,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AACvE,MAAM,UAAU,QAAQ,CACtB,GAAW,EACX,IAAY,EACZ,UAAkB,EAClB,UAA8B;IAE9B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CACb,+FAA+F,CAChG,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7D,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,eAAe,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,WAAW,KAAK,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IACvC,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAC5D,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;YACxE,IAAI,OAAO,IAAI,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACjF,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,MAAM,GAA4B,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5D,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;gBAChB,IACE,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,CAAC;oBACrD,MAAM,CAAC,QAAQ,KAAK,SAAS,EAC7B,CAAC;oBACD,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;gBACzB,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS;oBAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACrF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CACb,qHAAqH,CACtH,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1F,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,OAAO,IAAI,IAAI,EAAE,OAAO,CAAC;IACnD,MAAM,OAAO,GAAG,WAAW;QACzB,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACzB,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CAAC,YAAY,MAAM,sDAAsD,CAAC,CAAC;YAC5F,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IAEjB,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;IACpC,MAAM,cAAc,GAClB,OAAO,CAAC,eAAe,KAAK,SAAS;QACnC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,eAAe,EAAE,yBAAyB,CAAC,CAAC;IACtE,IAAI,cAAc,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CACb,8CAA8C,OAAO,CAAC,eAAe,iBAAiB,CACvF,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GACf,OAAO,CAAC,YAAY,KAAK,SAAS;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC;IAChE,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,YAAY,iBAAiB,CACjF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,eAAe,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC;QAClE,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CACb,2CAA2C,OAAO,CAAC,YAAY,KAAK,MAAM,CAAC,KAAK,EAAE,CACnF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACrD,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,uCAAuC,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAI,KAAK,CACb,uCAAuC,KAAK,iCAAiC,CAC9E,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC,CAAC,CAAC;IAEH,IAAI,KAAmC,CAAC;IACxC,IAAI,CAAC;QACH,KAAK,GAAG,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,iBAAiB,SAAS,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvD,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,IAAI,EAAE,WAAW,CAAC;IAC3D,MAAM,eAAe,GAAG,KAAK,CAAC,gBAAgB,IAAI,IAAI,EAAE,gBAAgB,CAAC;IACzE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC;IAC7C,MAAM,kBAAkB,GAAG,KAAK,CAAC,oBAAoB,IAAI,SAAS,CAAC,oBAAoB,CAAC;IACxF,OAAO;QACL,IAAI;QACJ,GAAG;QACH,IAAI;QACJ,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,CAAC;QACjD,GAAG,CAAC,eAAe,KAAK,SAAS,IAAI,EAAE,eAAe,EAAE,CAAC;QACzD,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE;QACpC,OAAO;QACP,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI,YAAY;QAC9C,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,QAAQ,EAAE,KAAK,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,IAAI,iBAAiB;QACrE,cAAc,EAAE,KAAK,CAAC,eAAe,IAAI,SAAS,CAAC,eAAe,IAAI,uBAAuB;QAC7F,YAAY,EAAE,KAAK,CAAC,aAAa,IAAI,SAAS,CAAC,aAAa,IAAI,EAAE;QAClE,GAAG,CAAC,kBAAkB,KAAK,SAAS,IAAI,EAAE,kBAAkB,EAAE,CAAC;QAC/D,GAAG,EAAE,EAAE,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE;QACvC,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE;YACP,GAAG,CAAC,cAAc,IAAI,EAAE,cAAc,EAAE,CAAC;YACzC,GAAG,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,CAAC;YACnC,OAAO;SACR;QACD,OAAO;QACP,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,KAAK,CAAC,QAAQ;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAChB,KAAkE;IAElE,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACtD,OAAO,KAAK,CAAC,WAAW,KAAK,SAAS;QACpC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE;QACtB,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GACX,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAChG,OAAO,OAAO,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,SAAS,CAAC;AACrE,CAAC;AAED,SAAS,QAAQ,CACf,MAA+B,EAC/B,IAAY,EACZ,KAAa,EACb,GAAW;IAEX,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,CAAC,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;IAC3B,MAAM,IAAI,GAAG;QACX,IAAI;QACJ,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;KACrD,CAAC;IACF,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,MAAM,MAAM,GAAqB,MAAM,CAAC,MAAM,CAAC;YAC/C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,iBAAiB,OAAO,EAAE,CAAC,CAAC;YAC3E,CAAC;YACD,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM;aACP,CAAC;QACJ,CAAC;QACD,KAAK,WAAW;YACd,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,oBAAoB,CAAC,CAAC;YACxD,CAAC;YACD,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC;gBAC3E,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;aACrD,CAAC;QACJ,KAAK,YAAY;YACf,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,YAAY;gBAClB,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC;gBAChC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;aAC/B,CAAC;QACJ,KAAK,aAAa;YAChB,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;QACpF,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,KAAK,GAAqB,MAAM,CAAC,KAAK,CAAC;YAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,OAAO,GAAG,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,OAAO;oBAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,gBAAgB,OAAO,EAAE,CAAC,CAAC;YAC1E,CAAC;YACD,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC;QACpE,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,aAAa,EAAE,GAAG,KAAK,gBAAgB,CAAC,CAAC;YACzE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,MAAM,IAAI,KAAK,CACb,WAAW,KAAK,mBAAmB,MAAM,CAAC,aAAa,8CAA8C,CACtG,CAAC;YACJ,CAAC;YACD,MAAM,MAAM,GAAG,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;YAC3D,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,mBAAmB,MAAM,CAAC,aAAa,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9F,CAAC;YACD,OAAO;gBACL,GAAG,IAAI;gBACP,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,MAAM,CAAC,aAAa;gBAClC,QAAQ,EAAE,MAAM,CAAC,QAAQ;aAC1B,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/plugin-evals/tools.d.ts

@@ -0,0 +1,42 @@
+/** The tools a case's `allowed_tools` grants by itself, any other needs a grant of the run */
+export declare const READ_ONLY_TOOLS: ReadonlySet<string>;
+/** The workspace tools an eval run can give, in the order the agent gets them */
+export declare const WORKSPACE_TOOLS: readonly ["read_file", "list_files", "glob", "grep", "write_file", "edit_file", "run_command"];
+export type WorkspaceTool = (typeof WORKSPACE_TOOLS)[number];
+/** A tool grant: a permission rule such as `Write` or `Bash(npm test *)` */
+export interface ToolGrant {
+    rule: string;
+    tool: string;
+    specifier?: string;
+}
+/** A grant read from its rule text, or why the text is not a rule */
+export declare function parseGrant(rule: string): ToolGrant | {
+    error: string;
+};
+/** Claude Code's name of a Cogitator tool, the tool's own name when it has no counterpart */
+export declare function claudeNameOf(tool: string): string;
+/** Whether a grant of `grantTool` covers the tool named `tool` */
+export declare function grantCovers(grantTool: string, tool: string): boolean;
+/** Whether `name` (a grader's `tool`) names the call, by Claude Code's name or Cogitator's own */
+export declare function namesCall(name: string, call: {
+    name: string;
+    tool: string;
+}): boolean;
+/** The tools a case may use, and what it asked for that the run does not give */
+export interface CaseGrants {
+    grants: ToolGrant[];
+    /** Tools the case lists that are neither read-only nor granted for the run */
+    notGranted: string[];
+    /** Granted Claude Code tools that have no Cogitator counterpart in an eval run */
+    noCounterpart: string[];
+}
+/**
+ * What a case may use: the read-only tools its `allowed_tools` lists, and every grant of the run.
+ * A rule that cannot be read is an error
+ */
+export declare function caseGrants(allowedTools: readonly string[], runGrants: readonly ToolGrant[]): CaseGrants | {
+    error: string;
+};
+/** Whether any grant covers the tool named `tool` */
+export declare function isGranted(grants: readonly ToolGrant[], tool: string): boolean;
+//# sourceMappingURL=tools.d.ts.map

package/dist/plugin-evals/tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/tools.ts"],"names":[],"mappings":"AAyCA,8FAA8F;AAC9F,eAAO,MAAM,eAAe,EAAE,WAAW,CAAC,MAAM,CAc9C,CAAC;AAkBH,iFAAiF;AACjF,eAAO,MAAM,eAAe,gGAQlB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D,4EAA4E;AAC5E,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAStE;AAED,6FAA6F;AAC7F,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEjD;AAED,kEAAkE;AAClE,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAIpE;AAED,kGAAkG;AAClG,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,CAGrF;AAED,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,8EAA8E;IAC9E,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,kFAAkF;IAClF,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;;GAGG;AACH,wBAAgB,UAAU,CACxB,YAAY,EAAE,SAAS,MAAM,EAAE,EAC/B,SAAS,EAAE,SAAS,SAAS,EAAE,GAC9B,UAAU,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAiBhC;AAED,qDAAqD;AACrD,wBAAgB,SAAS,CAAC,MAAM,EAAE,SAAS,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7E"}