@cogitator-ai/evals 0.13.2 → 0.14.0

package/dist/plugin-evals/schema.d.ts

@@ -0,0 +1,179 @@
+import { z } from 'zod';
+/** Default runs per arm, as Claude Code's `runs` */
+export declare const DEFAULT_RUNS = 3;
+/** Default turn cap of a run */
+export declare const DEFAULT_MAX_TURNS = 10;
+/** Default wall-clock cap of a run, in seconds */
+export declare const DEFAULT_TIMEOUT_SECONDS = 300;
+/** The frontmatter of a case's `prompt.md`: an unknown key is an error */
+export declare const PromptFrontmatterSchema: z.ZodObject<{
+    model: z.ZodOptional<z.ZodString>;
+    max_turns: z.ZodOptional<z.ZodNumber>;
+    timeout_seconds: z.ZodOptional<z.ZodNumber>;
+    allowed_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    append_system_prompt: z.ZodOptional<z.ZodString>;
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>, z.ZodTransform<string, string | number | boolean>>>>;
+    name: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    plugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    runs: z.ZodOptional<z.ZodNumber>;
+    expected_outcome: z.ZodOptional<z.ZodString>;
+    schema_version: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"1.1">, z.ZodLiteral<1.1>]>>;
+}, z.core.$strict>;
+/** A case's `case.yaml` */
+export declare const CaseYamlSchema: z.ZodObject<{
+    name: z.ZodString;
+    execution: z.ZodOptional<z.ZodObject<{
+        prompt: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+        max_turns: z.ZodOptional<z.ZodNumber>;
+        timeout_seconds: z.ZodOptional<z.ZodNumber>;
+        allowed_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        append_system_prompt: z.ZodOptional<z.ZodString>;
+        env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>, z.ZodTransform<string, string | number | boolean>>>>;
+    }, z.core.$strict>>;
+    context: z.ZodOptional<z.ZodObject<{
+        scaffold_script: z.ZodOptional<z.ZodString>;
+        history_file: z.ZodOptional<z.ZodString>;
+        add_dirs: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    }, z.core.$strict>>;
+    graders: z.ZodOptional<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    description: z.ZodOptional<z.ZodString>;
+    tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    plugins: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    runs: z.ZodOptional<z.ZodNumber>;
+    expected_outcome: z.ZodOptional<z.ZodString>;
+    schema_version: z.ZodUnion<readonly [z.ZodLiteral<"1.1">, z.ZodLiteral<1.1>]>;
+}, z.core.$strict>;
+export type PromptFrontmatter = z.output<typeof PromptFrontmatterSchema>;
+export type CaseYaml = z.output<typeof CaseYamlSchema>;
+/** A grader's options, from a `graders/*.md` frontmatter (with its body) or a `case.yaml` entry */
+export declare const GraderSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"regex">;
+    pattern: z.ZodString;
+    flags: z.ZodDefault<z.ZodString>;
+    match: z.ZodDefault<z.ZodString>;
+    target: z.ZodDefault<z.ZodUnion<readonly [z.ZodEnum<{
+        last_message: "last_message";
+        trace: "trace";
+        files: "files";
+        mock_calls: "mock_calls";
+    }>, z.ZodObject<{
+        source: z.ZodLiteral<"file">;
+        path: z.ZodString;
+    }, z.core.$strict>]>>;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"tool_used">;
+    tool: z.ZodString;
+    input_match: z.ZodOptional<z.ZodString>;
+    min: z.ZodDefault<z.ZodNumber>;
+    max: z.ZodOptional<z.ZodNumber>;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"tool_order">;
+    before: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        tool: z.ZodString;
+        input_match: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>]>;
+    after: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
+        tool: z.ZodString;
+        input_match: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>]>;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"file_exists">;
+    path: z.ZodString;
+    exists: z.ZodDefault<z.ZodBoolean>;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"llm">;
+    criteria: z.ZodString;
+    focus: z.ZodDefault<z.ZodUnion<readonly [z.ZodEnum<{
+        last_message: "last_message";
+        trace: "trace";
+        files: "files";
+        mock_calls: "mock_calls";
+    }>, z.ZodObject<{
+        source: z.ZodLiteral<"file">;
+        path: z.ZodString;
+    }, z.core.$strict>]>>;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>, z.ZodObject<{
+    type: z.ZodLiteral<"baseline">;
+    baseline_file: z.ZodString;
+    criteria: z.ZodString;
+    name: z.ZodOptional<z.ZodString>;
+    weight: z.ZodDefault<z.ZodNumber>;
+    arm: z.ZodOptional<z.ZodEnum<{
+        "with-only": "with-only";
+        both: "both";
+    }>>;
+}, z.core.$strict>], "type">;
+export type GraderInput = z.output<typeof GraderSchema>;
+/** The frontmatter of a mocked tool, `mocks/<server>/<tool>.md` */
+export declare const MockFrontmatterSchema: z.ZodObject<{
+    type: z.ZodDefault<z.ZodEnum<{
+        fixed: "fixed";
+        agent: "agent";
+    }>>;
+    expect: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    error: z.ZodDefault<z.ZodBoolean>;
+    abort_when: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+/** A server's `_tools.json`: a saved `tools/list` response, or its `tools` array */
+export declare const ToolsListSchema: z.ZodUnion<readonly [z.ZodObject<{
+    tools: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+    }, z.core.$loose>>;
+}, z.core.$strip>, z.ZodArray<z.ZodObject<{
+    name: z.ZodString;
+}, z.core.$loose>>]>;
+/** Zod issues in one line: `path: message, path: message` */
+export declare function issuesText(error: z.ZodError): string;
+/** A Markdown file split into its YAML frontmatter (empty without one) and its body */
+export interface MarkdownParts {
+    fields: Record<string, unknown>;
+    body: string;
+}
+/**
+ * The frontmatter between the first two `---` lines of `text` read as a YAML mapping, and the
+ * body after it. A file without frontmatter is all body
+ */
+export declare function readMarkdown(text: string): MarkdownParts | {
+    error: string;
+};
+/** `text` read as YAML, or why it is not valid YAML */
+export declare function readYaml(text: string): {
+    value: unknown;
+} | {
+    error: string;
+};
+export declare function isRecord(value: unknown): value is Record<string, unknown>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/plugin-evals/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,oDAAoD;AACpD,eAAO,MAAM,YAAY,IAAI,CAAC;AAC9B,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,KAAK,CAAC;AACpC,kDAAkD;AAClD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AA0C3C,0EAA0E;AAC1E,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBAIlC,CAAC;AAEH,2BAA2B;AAC3B,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;kBAazB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACzE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AAiCvD,mGAAmG;AACnG,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAmDxB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD,mEAAmE;AACnE,eAAO,MAAM,qBAAqB;;;;;;;;kBAKhC,CAAC;AAEH,oFAAoF;AACpF,eAAO,MAAM,eAAe;;;;;;oBAG1B,CAAC;AAEH,6DAA6D;AAC7D,wBAAgB,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAOpD;AAED,uFAAuF;AACvF,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAa5E;AAED,uDAAuD;AACvD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAS7E;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEzE"}

package/dist/plugin-evals/schema.js

@@ -0,0 +1,194 @@
+import { parseDocument } from 'yaml';
+import { z } from 'zod';
+/** Default runs per arm, as Claude Code's `runs` */
+export const DEFAULT_RUNS = 3;
+/** Default turn cap of a run */
+export const DEFAULT_MAX_TURNS = 10;
+/** Default wall-clock cap of a run, in seconds */
+export const DEFAULT_TIMEOUT_SECONDS = 300;
+const ENV_KEY = /^EVAL_[A-Z0-9_]*$/;
+const schemaVersion = z.union([z.literal('1.1'), z.literal(1.1)], {
+    error: 'must be "1.1"',
+});
+const nonEmpty = z.string().trim().min(1, 'must not be empty');
+const envSchema = z
+    .record(z.string(), z.union([z.string(), z.number(), z.boolean()]).transform(String))
+    .superRefine((env, context) => {
+    for (const key of Object.keys(env)) {
+        if (!ENV_KEY.test(key)) {
+            context.addIssue({
+                code: 'custom',
+                path: [key],
+                message: 'environment keys must match EVAL_[A-Z0-9_]*',
+            });
+        }
+    }
+});
+const runFields = {
+    model: nonEmpty.optional(),
+    max_turns: z.number().int().min(1).max(200).optional(),
+    timeout_seconds: z.number().positive().max(3600).optional(),
+    allowed_tools: z.array(nonEmpty).optional(),
+    append_system_prompt: z.string().optional(),
+    env: envSchema.optional(),
+};
+const caseFields = {
+    name: nonEmpty.optional(),
+    description: z.string().optional(),
+    tags: z.array(nonEmpty).optional(),
+    plugins: z.array(nonEmpty).optional(),
+    runs: z.number().int().min(1).max(50).optional(),
+    expected_outcome: z.string().optional(),
+};
+/** The frontmatter of a case's `prompt.md`: an unknown key is an error */
+export const PromptFrontmatterSchema = z.strictObject({
+    schema_version: schemaVersion.optional(),
+    ...caseFields,
+    ...runFields,
+});
+/** A case's `case.yaml` */
+export const CaseYamlSchema = z.strictObject({
+    schema_version: schemaVersion,
+    ...caseFields,
+    name: nonEmpty,
+    execution: z.strictObject({ ...runFields, prompt: z.string().optional() }).optional(),
+    context: z
+        .strictObject({
+        scaffold_script: nonEmpty.optional(),
+        history_file: nonEmpty.optional(),
+        add_dirs: z.array(nonEmpty).optional(),
+    })
+        .optional(),
+    graders: z.array(z.record(z.string(), z.unknown())).optional(),
+});
+const regexSource = z.string().refine((source) => {
+    try {
+        new RegExp(source);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}, { error: 'is not a valid JavaScript regular expression (put case-insensitivity in flags: i)' });
+const targetSchema = z.union([
+    z.enum(['last_message', 'trace', 'files', 'mock_calls']),
+    z.strictObject({ source: z.literal('file'), path: nonEmpty }),
+], { error: 'must be last_message, trace, files, mock_calls or { source: file, path: <path> }' });
+const toolMatcher = z.union([
+    nonEmpty,
+    z.strictObject({ tool: nonEmpty, input_match: regexSource.optional() }),
+]);
+const graderBase = {
+    name: nonEmpty.optional(),
+    weight: z.number().positive().default(1),
+    arm: z.enum(['with-only', 'both']).optional(),
+};
+/** A grader's options, from a `graders/*.md` frontmatter (with its body) or a `case.yaml` entry */
+export const GraderSchema = z.discriminatedUnion('type', [
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('regex'),
+        pattern: regexSource,
+        flags: z
+            .string()
+            .regex(/^[imsu]*$/, 'may only hold the flags i, m, s and u')
+            .default(''),
+        match: z
+            .string()
+            .regex(/^(?:contains|not_contains|count:\d+)$/, 'must be contains, not_contains or count:N')
+            .default('contains'),
+        target: targetSchema.default('last_message'),
+    }),
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('tool_used'),
+        tool: nonEmpty,
+        input_match: regexSource.optional(),
+        min: z.number().int().min(0).default(1),
+        max: z.number().int().min(0).optional(),
+    }),
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('tool_order'),
+        before: toolMatcher,
+        after: toolMatcher,
+    }),
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('file_exists'),
+        path: nonEmpty,
+        exists: z.boolean().default(true),
+    }),
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('llm'),
+        criteria: nonEmpty,
+        focus: targetSchema.default('last_message'),
+    }),
+    z.strictObject({
+        ...graderBase,
+        type: z.literal('baseline'),
+        baseline_file: nonEmpty,
+        criteria: nonEmpty,
+    }),
+], { error: 'type must be one of regex, tool_used, tool_order, file_exists, llm, baseline' });
+/** The frontmatter of a mocked tool, `mocks/<server>/<tool>.md` */
+export const MockFrontmatterSchema = z.strictObject({
+    type: z.enum(['fixed', 'agent']).default('fixed'),
+    expect: z.record(z.string(), z.unknown()).optional(),
+    error: z.boolean().default(false),
+    abort_when: z.string().optional(),
+});
+/** A server's `_tools.json`: a saved `tools/list` response, or its `tools` array */
+export const ToolsListSchema = z.union([
+    z.object({ tools: z.array(z.looseObject({ name: nonEmpty })) }),
+    z.array(z.looseObject({ name: nonEmpty })),
+]);
+/** Zod issues in one line: `path: message, path: message` */
+export function issuesText(error) {
+    return error.issues
+        .map((issue) => {
+        const path = issue.path.map(String).join('.');
+        return path === '' ? issue.message : `${path}: ${issue.message}`;
+    })
+        .join(', ');
+}
+/**
+ * The frontmatter between the first two `---` lines of `text` read as a YAML mapping, and the
+ * body after it. A file without frontmatter is all body
+ */
+export function readMarkdown(text) {
+    const source = text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+    const lines = source.split('\n');
+    if (lines[0]?.trimEnd() !== '---')
+        return { fields: {}, body: source };
+    const end = lines.findIndex((line, index) => index > 0 && line.trimEnd() === '---');
+    if (end === -1)
+        return { error: 'the frontmatter has no closing --- line' };
+    const parsed = readYaml(lines.slice(1, end).join('\n'));
+    if ('error' in parsed)
+        return parsed;
+    if (parsed.value === null || parsed.value === undefined) {
+        return { fields: {}, body: lines.slice(end + 1).join('\n') };
+    }
+    if (!isRecord(parsed.value))
+        return { error: 'the frontmatter is not a YAML mapping' };
+    return { fields: parsed.value, body: lines.slice(end + 1).join('\n') };
+}
+/** `text` read as YAML, or why it is not valid YAML */
+export function readYaml(text) {
+    try {
+        const document = parseDocument(text, { prettyErrors: false });
+        const [problem] = document.errors;
+        if (problem)
+            return { error: `invalid YAML: ${problem.message}` };
+        return { value: document.toJS({ maxAliasCount: 100 }) };
+    }
+    catch (error) {
+        return { error: `invalid YAML: ${error instanceof Error ? error.message : String(error)}` };
+    }
+}
+export function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+//# sourceMappingURL=schema.js.map

package/dist/plugin-evals/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/plugin-evals/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,oDAAoD;AACpD,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAC9B,gCAAgC;AAChC,MAAM,CAAC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AACpC,kDAAkD;AAClD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C,MAAM,OAAO,GAAG,mBAAmB,CAAC;AAEpC,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE;IAChE,KAAK,EAAE,eAAe;CACvB,CAAC,CAAC;AAEH,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,mBAAmB,CAAC,CAAC;AAE/D,MAAM,SAAS,GAAG,CAAC;KAChB,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;KACpF,WAAW,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAC5B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,QAAQ,CAAC;gBACf,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,GAAG,CAAC;gBACX,OAAO,EAAE,6CAA6C;aACvD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,SAAS,GAAG;IAChB,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE;IAC1B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;IACtD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;IAC3D,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE;IAC3C,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3C,GAAG,EAAE,SAAS,CAAC,QAAQ,EAAE;CAC1B,CAAC;AAEF,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE;IACzB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE;IAClC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE;IACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;IAChD,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACxC,CAAC;AAEF,0EAA0E;AAC1E,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,YAAY,CAAC;IACpD,cAAc,EAAE,aAAa,CAAC,QAAQ,EAAE;IACxC,GAAG,UAAU;IACb,GAAG,SAAS;CACb,CAAC,CAAC;AAEH,2BAA2B;AAC3B,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,YAAY,CAAC;IAC3C,cAAc,EAAE,aAAa;IAC7B,GAAG,UAAU;IACb,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,CAAC,CAAC,YAAY,CAAC,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrF,OAAO,EAAE,CAAC;SACP,YAAY,CAAC;QACZ,eAAe,EAAE,QAAQ,CAAC,QAAQ,EAAE;QACpC,YAAY,EAAE,QAAQ,CAAC,QAAQ,EAAE;QACjC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE;KACvC,CAAC;SACD,QAAQ,EAAE;IACb,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC/D,CAAC,CAAC;AAKH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,MAAM,CACnC,CAAC,MAAM,EAAE,EAAE;IACT,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC,EACD,EAAE,KAAK,EAAE,mFAAmF,EAAE,CAC/F,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAC1B;IACE,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;CAC9D,EACD,EAAE,KAAK,EAAE,kFAAkF,EAAE,CAC9F,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;IAC1B,QAAQ;IACR,CAAC,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC;CACxE,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG;IACjB,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE;IACzB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;CAC9C,CAAC;AAEF,mGAAmG;AACnG,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,kBAAkB,CAC9C,MAAM,EACN;IACE,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;QACxB,OAAO,EAAE,WAAW;QACpB,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,KAAK,CAAC,WAAW,EAAE,uCAAuC,CAAC;aAC3D,OAAO,CAAC,EAAE,CAAC;QACd,KAAK,EAAE,CAAC;aACL,MAAM,EAAE;aACR,KAAK,CAAC,uCAAuC,EAAE,2CAA2C,CAAC;aAC3F,OAAO,CAAC,UAAU,CAAC;QACtB,MAAM,EAAE,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC;KAC7C,CAAC;IACF,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC;QAC5B,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE;QACnC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QACvC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KACxC,CAAC;IACF,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC;QAC7B,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,WAAW;KACnB,CAAC;IACF,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC;QAC9B,IAAI,EAAE,QAAQ;QACd,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;KAClC,CAAC;IACF,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QACtB,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC;KAC5C,CAAC;IACF,CAAC,CAAC,YAAY,CAAC;QACb,GAAG,UAAU;QACb,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3B,aAAa,EAAE,QAAQ;QACvB,QAAQ,EAAE,QAAQ;KACnB,CAAC;CACH,EACD,EAAE,KAAK,EAAE,8EAA8E,EAAE,CAC1F,CAAC;AAIF,mEAAmE;AACnE,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,YAAY,CAAC;IAClD,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IACjD,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACpD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACjC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAEH,oFAAoF;AACpF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC;IAC/D,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;CAC3C,CAAC,CAAC;AAEH,6DAA6D;AAC7D,MAAM,UAAU,UAAU,CAAC,KAAiB;IAC1C,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9C,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;IACnE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAQD;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,KAAK;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IACvE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,KAAK,CAAC,CAAC;IACpF,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,yCAAyC,EAAE,CAAC;IAC5E,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,IAAI,OAAO,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IACrC,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACxD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,uCAAuC,EAAE,CAAC;IACvF,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACzE,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;QAClC,IAAI,OAAO;YAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAClE,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,CAAY,EAAE,CAAC;IACrE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,KAAK,EAAE,iBAAiB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IAC9F,CAAC;AACH,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}

package/dist/plugin-evals/suite.d.ts

@@ -0,0 +1,40 @@
+import type { PluginEvalCase } from './types.js';
+/** A plugin eval run that cannot start: no plugin, no cases, a bad option */
+export declare class PluginEvalError extends Error {
+    constructor(message: string);
+}
+/** A case directory, loaded or with why it did not load */
+export interface SuiteCase {
+    name: string;
+    /** The case directory relative to the eval directory */
+    path: string;
+    case?: PluginEvalCase;
+    error?: string;
+}
+/** The plugin under test and the cases to run */
+export interface LoadedSuite {
+    pluginRoot: string;
+    evalDir: string;
+    /** Where `results/<timestamp>/` goes */
+    resultsDir: string;
+    /** The target named an installed plugin instead of a path */
+    named: boolean;
+    cases: SuiteCase[];
+    warnings: string[];
+}
+export interface SuiteOptions {
+    target?: string;
+    cwd: string;
+    evalDir?: string;
+    caseGlob?: string;
+    tags?: readonly string[];
+    /** The install path of an installed plugin by `name` or `name@marketplace`, if there is one */
+    installed: (id: string) => Promise<string | undefined>;
+}
+/** Whether `dir` holds a plugin manifest Cogitator reads */
+export declare function isPluginRoot(dir: string): boolean;
+/** Resolves the target, finds the eval directory and loads every case the filters keep */
+export declare function loadSuite(options: SuiteOptions): Promise<LoadedSuite>;
+/** Loads one case: its prompt, settings, graders, context and mocks */
+export declare function loadCase(dir: string, path: string, pluginRoot: string, suiteMocks: string | undefined): PluginEvalCase;
+//# sourceMappingURL=suite.d.ts.map

package/dist/plugin-evals/suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../src/plugin-evals/suite.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EACV,cAAc,EAIf,MAAM,SAAS,CAAC;AAEjB,6EAA6E;AAC7E,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,OAAO,EAAE,MAAM;CAI5B;AAMD,2DAA2D;AAC3D,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACzB,+FAA+F;IAC/F,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;CACxD;AAED,4DAA4D;AAC5D,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEjD;AAwDD,0FAA0F;AAC1F,wBAAsB,SAAS,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CA2F3E;AAqGD,uEAAuE;AACvE,wBAAgB,QAAQ,CACtB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GAAG,SAAS,GAC7B,cAAc,CA0IhB"}