@plumloom/cli 0.1.0

package/dist/guide/eval-creation-backend.js

@@ -0,0 +1,16 @@
+import { getModels } from '../actions/models.js';
+import { listWorkspaces } from '../actions/workspaces.js';
+export class ActionEvalCreationBackend {
+    #actions;
+    constructor(input) {
+        this.#actions = input.actions;
+    }
+    async listWorkspaces(signal) {
+        const page = await listWorkspaces(this.#actions, {}, signal);
+        return page.items;
+    }
+    listModels(signal) {
+        return getModels(this.#actions, signal);
+    }
+}
+//# sourceMappingURL=eval-creation-backend.js.map

package/dist/guide/eval-creation-backend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-creation-backend.js","sourceRoot":"","sources":["../../src/guide/eval-creation-backend.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAa1D,MAAM,OAAO,yBAAyB;IAC3B,QAAQ,CAAgB;IAEjC,YAAY,KAAiC;QAC3C,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAoB;QACvC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;QAC7D,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,UAAU,CAAC,MAAoB;QAC7B,OAAO,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;CACF"}

package/dist/guide/eval-guide.d.ts

@@ -0,0 +1,24 @@
+import type { EvalCreationBackend } from './eval-creation-backend.js';
+export type EvalGuideOutput = {
+    /** Text to show the user. */
+    output: string;
+    /** Deterministic CLI command the caller should execute now, when confirmed. */
+    command?: string;
+};
+/** True when the user is asking to run or set up evaluations from files. */
+export declare function isEvalFileRequest(message: string): boolean;
+export declare class EvalGuideFlow {
+    #private;
+    constructor(dependencies: {
+        backend: EvalCreationBackend;
+        signal?: AbortSignal;
+    });
+    get isComplete(): boolean;
+    /** Evaluation ids are created by the deterministic command, not by the guide. */
+    readonly createdEvaluationId: string;
+    /** The eval file the user pointed at, once one has been accepted. */
+    get savedDraftPath(): string;
+    start(message: string): Promise<EvalGuideOutput>;
+    handle(message: string): Promise<EvalGuideOutput>;
+}
+//# sourceMappingURL=eval-guide.d.ts.map

package/dist/guide/eval-guide.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-guide.d.ts","sourceRoot":"","sources":["../../src/guide/eval-guide.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAetE,MAAM,MAAM,eAAe,GAAG;IAC5B,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAcF,4EAA4E;AAC5E,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAI1D;AAuCD,qBAAa,aAAa;;gBAWZ,YAAY,EAAE;QAAE,OAAO,EAAE,mBAAmB,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE;IAKhF,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,iFAAiF;IACjF,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAM;IAE1C,qEAAqE;IACrE,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiBhD,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;CA0IlD"}

package/dist/guide/eval-guide.js

@@ -0,0 +1,219 @@
+import { basename } from 'node:path';
+import { readEvalFile } from '../configured-run/file-inputs.js';
+import { parseConfiguredRunInput, validateConfiguredRunInput, } from '../configured-run/validation.js';
+import { AutoevalError } from '../errors/autoeval-error.js';
+import { renderRule } from '../output/layout.js';
+import { readSuiteManifest } from '../suite/manifest.js';
+const SUITE_ANSWERS = new Set(['suite', 'many', 'multiple', 'manifest', '2']);
+const ONE_ANSWERS = new Set(['one', 'single', 'a single eval', '1']);
+const YES_ANSWERS = new Set(['y', 'yes', 'run', 'run it', 'ok', 'go']);
+const RUN_FILE_PATTERNS = [
+    /\b(?:run|execute|start)\b[^.!?]*\b(?:an?|my|the|this)?\s*(?:eval|evaluation|suite)\b/iu,
+    /\b(?:how do i|help me)\b[^.!?]*\b(?:run|use)\b[^.!?]*\b(?:eval|evaluation|suite)\b/iu,
+    /\b(?:create|author|build|make|set ?up)\b[^.!?]*\b(?:an?|my|the|new)?\s*(?:eval|evaluation)\b/iu,
+];
+const UUID_PATTERN = /\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/iu;
+/** True when the user is asking to run or set up evaluations from files. */
+export function isEvalFileRequest(message) {
+    if (/^\s*autoeval\b/iu.test(message))
+        return false;
+    if (UUID_PATTERN.test(message))
+        return false;
+    return RUN_FILE_PATTERNS.some((pattern) => pattern.test(message));
+}
+/** A quoted or bare path mentioned in the message, if any. */
+function extractPath(message) {
+    const quoted = /"([^"]+)"|'([^']+)'/u.exec(message);
+    const candidate = quoted?.[1] ??
+        quoted?.[2] ??
+        message.split(/\s+/u).find((token) => /[./\\]/u.test(token) && !token.endsWith('.'));
+    return candidate?.trim() === '' ? undefined : candidate;
+}
+function quoteCommandValue(value) {
+    return /\s/u.test(value) ? `"${value.replace(/["\\]/gu, '\\$&')}"` : value;
+}
+function summarize(path, validation) {
+    const contextLabel = validation.contextType === 'agent_trace'
+        ? 'Agent trace'
+        : validation.contextType === 'conversation'
+            ? 'Conversation'
+            : 'Scenario';
+    const unit = validation.contextType === 'scenario'
+        ? 'scenarios'
+        : validation.contextType === 'conversation'
+            ? 'messages'
+            : 'resource spans';
+    const lines = [
+        `${basename(path)} — ${contextLabel} · ${validation.artifactCount} ${unit}`,
+        validation.models.primary
+            ? `Model under test ${validation.models.primary.displayName} · judge ${validation.models.judge.displayName}`
+            : `Judge ${validation.models.judge.displayName}`,
+    ];
+    lines.push('Valid.');
+    return lines.join('\n');
+}
+export class EvalGuideFlow {
+    #backend;
+    #signal;
+    #step = 'scope';
+    #scope;
+    #evalFile = '';
+    #manifestFile = '';
+    #workspaceId = '';
+    #workspaceChoices = [];
+    #command = '';
+    constructor(dependencies) {
+        this.#backend = dependencies.backend;
+        this.#signal = dependencies.signal;
+    }
+    get isComplete() {
+        return this.#step === 'done';
+    }
+    /** Evaluation ids are created by the deterministic command, not by the guide. */
+    createdEvaluationId = '';
+    /** The eval file the user pointed at, once one has been accepted. */
+    get savedDraftPath() {
+        return this.#evalFile;
+    }
+    start(message) {
+        const inlinePath = extractPath(message);
+        if (inlinePath !== undefined) {
+            return this.#acceptPath(inlinePath);
+        }
+        if (/\bsuite\b|\bmanifest\b/iu.test(message)) {
+            this.#scope = 'suite';
+            this.#step = 'file';
+            return Promise.resolve({ output: 'Which suite manifest? (path to the YAML or JSON file)' });
+        }
+        return Promise.resolve({
+            output: 'Evaluations are defined in files, so I just need the file.\n' +
+                'Is this one eval, or a suite?  [one | suite]',
+        });
+    }
+    handle(message) {
+        const answer = message.trim();
+        const lowered = answer.toLowerCase();
+        if (lowered === 'cancel' || lowered === 'quit' || lowered === 'exit') {
+            this.#step = 'done';
+            return Promise.resolve({ output: 'Cancelled.' });
+        }
+        switch (this.#step) {
+            case 'scope':
+                return this.#answerScope(answer, lowered);
+            case 'file':
+                return this.#acceptPath(extractPath(answer) ?? answer);
+            case 'workspace':
+                return Promise.resolve(this.#answerWorkspace(answer));
+            case 'confirm':
+                return Promise.resolve(this.#answerConfirm(lowered));
+            default:
+                this.#step = 'done';
+                return Promise.resolve({ output: 'Nothing left to do.' });
+        }
+    }
+    async #answerScope(answer, lowered) {
+        const path = extractPath(answer);
+        if (path !== undefined)
+            return this.#acceptPath(path);
+        if (SUITE_ANSWERS.has(lowered)) {
+            this.#scope = 'suite';
+            this.#step = 'file';
+            return { output: 'Which suite manifest? (path to the YAML or JSON file)' };
+        }
+        if (ONE_ANSWERS.has(lowered)) {
+            this.#scope = 'one';
+            this.#step = 'file';
+            return { output: 'Which eval file? (path to the .autoeval.json file)' };
+        }
+        return { output: 'Answer `one` or `suite`, or give me the file path directly.' };
+    }
+    async #acceptPath(path) {
+        const isManifest = this.#scope === 'suite' || /\.(?:ya?ml)$/iu.test(path) || /suite/iu.test(basename(path));
+        return isManifest ? this.#acceptManifest(path) : this.#acceptEvalFile(path);
+    }
+    async #acceptEvalFile(path) {
+        const summary = await this.#describeEvalFile(path);
+        this.#scope = 'one';
+        this.#evalFile = path;
+        return this.#askWorkspaceOrConfirm(summary);
+    }
+    async #acceptManifest(path) {
+        const manifest = await readSuiteManifest(path);
+        const summaries = [];
+        for (const evalFile of manifest.evalFiles) {
+            summaries.push(await this.#describeEvalFile(evalFile));
+        }
+        this.#scope = 'suite';
+        this.#manifestFile = path;
+        this.#workspaceId = manifest.workspaceId;
+        this.#command = `autoeval suite run --manifest ${quoteCommandValue(path)}`;
+        this.#step = 'confirm';
+        return {
+            output: [
+                `${basename(path)} — ${manifest.evalFiles.length} evaluations · workspace ${manifest.workspaceId}`,
+                renderRule(),
+                summaries.join(`\n${renderRule()}\n`),
+                renderRule(),
+                this.#command,
+                'Run it now? [y/N]',
+            ].join('\n'),
+        };
+    }
+    async #describeEvalFile(path) {
+        const input = parseConfiguredRunInput(await readEvalFile(path));
+        const models = await this.#backend.listModels(this.#signal);
+        return summarize(path, validateConfiguredRunInput(input, models));
+    }
+    async #askWorkspaceOrConfirm(summary) {
+        if (this.#workspaceId === '') {
+            const workspaces = await this.#backend.listWorkspaces(this.#signal);
+            if (workspaces.length === 1 && workspaces[0]) {
+                this.#workspaceId = workspaces[0].id;
+            }
+            else {
+                this.#workspaceChoices = workspaces;
+                this.#step = 'workspace';
+                return {
+                    output: [
+                        summary,
+                        renderRule(),
+                        'Which workspace?',
+                        ...workspaces.map((workspace, index) => `  ${index + 1}. ${workspace.name}`),
+                    ].join('\n'),
+                };
+            }
+        }
+        return { output: `${summary}\n${renderRule()}\n${this.#buildConfirmation()}` };
+    }
+    #buildConfirmation() {
+        this.#command =
+            `autoeval eval create-from --workspace ${this.#workspaceId} ` +
+                `--input ${quoteCommandValue(this.#evalFile)} --run`;
+        this.#step = 'confirm';
+        return `${this.#command}\nRun it now? [y/N]`;
+    }
+    #answerWorkspace(answer) {
+        const index = Number.parseInt(answer, 10);
+        const chosen = this.#workspaceChoices[index - 1] ??
+            this.#workspaceChoices.find((workspace) => workspace.id === answer || workspace.name.toLowerCase() === answer.toLowerCase());
+        if (!chosen) {
+            throw new AutoevalError('Pick a workspace by number or name.', {
+                kind: 'usage',
+                code: 'WORKSPACE_CHOICE_UNKNOWN',
+            });
+        }
+        this.#workspaceId = chosen.id;
+        return { output: this.#buildConfirmation() };
+    }
+    #answerConfirm(lowered) {
+        this.#step = 'done';
+        if (YES_ANSWERS.has(lowered)) {
+            return {
+                output: `Running ${basename(this.#manifestFile || this.#evalFile)}...`,
+                command: this.#command,
+            };
+        }
+        return { output: `Run it when you are ready:\n${this.#command}` };
+    }
+}
+//# sourceMappingURL=eval-guide.js.map

package/dist/guide/eval-guide.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-guide.js","sourceRoot":"","sources":["../../src/guide/eval-guide.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAChE,OAAO,EACL,uBAAuB,EACvB,0BAA0B,GAE3B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAuBzD,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;AACnG,MAAM,WAAW,GAAwB,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC;AAC1F,MAAM,WAAW,GAAwB,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAE5F,MAAM,iBAAiB,GAAsB;IAC3C,wFAAwF;IACxF,sFAAsF;IACtF,gGAAgG;CACjG,CAAC;AAEF,MAAM,YAAY,GAAG,oEAAoE,CAAC;AAE1F,4EAA4E;AAC5E,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACnD,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,8DAA8D;AAC9D,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,MAAM,GAAG,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,SAAS,GACb,MAAM,EAAE,CAAC,CAAC,CAAC;QACX,MAAM,EAAE,CAAC,CAAC,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACvF,OAAO,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1D,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC7E,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,UAAyC;IACxE,MAAM,YAAY,GAChB,UAAU,CAAC,WAAW,KAAK,aAAa;QACtC,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,UAAU,CAAC,WAAW,KAAK,cAAc;YACzC,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,UAAU,CAAC;IACnB,MAAM,IAAI,GACR,UAAU,CAAC,WAAW,KAAK,UAAU;QACnC,CAAC,CAAC,WAAW;QACb,CAAC,CAAC,UAAU,CAAC,WAAW,KAAK,cAAc;YACzC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,gBAAgB,CAAC;IACzB,MAAM,KAAK,GAAG;QACZ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,YAAY,MAAM,UAAU,CAAC,aAAa,IAAI,IAAI,EAAE;QAC3E,UAAU,CAAC,MAAM,CAAC,OAAO;YACvB,CAAC,CAAC,oBAAoB,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,YAAY,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;YAC5G,CAAC,CAAC,SAAS,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE;KACnD,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,OAAO,aAAa;IACf,QAAQ,CAAsB;IAC9B,OAAO,CAA0B;IAC1C,KAAK,GAAS,OAAO,CAAC;IACtB,MAAM,CAA8B;IACpC,SAAS,GAAG,EAAE,CAAC;IACf,aAAa,GAAG,EAAE,CAAC;IACnB,YAAY,GAAG,EAAE,CAAC;IAClB,iBAAiB,GAAyB,EAAE,CAAC;IAC7C,QAAQ,GAAG,EAAE,CAAC;IAEd,YAAY,YAAoE;QAC9E,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,OAAO,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC;IAC/B,CAAC;IAED,iFAAiF;IACxE,mBAAmB,GAAW,EAAE,CAAC;IAE1C,qEAAqE;IACrE,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,OAAe;QACnB,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,0BAA0B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,uDAAuD,EAAE,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC;YACrB,MAAM,EACJ,8DAA8D;gBAC9D,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,OAAe;QACpB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;QACrC,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,MAAM,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACrE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QACnD,CAAC;QAED,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACnB,KAAK,OAAO;gBACV,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,CAAC;YACzD,KAAK,WAAW;gBACd,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;YACxD,KAAK,SAAS;gBACZ,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;YACvD;gBACE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;gBACpB,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAe;QAChD,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,EAAE,MAAM,EAAE,uDAAuD,EAAE,CAAC;QAC7E,CAAC;QACD,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;YACpB,OAAO,EAAE,MAAM,EAAE,oDAAoD,EAAE,CAAC;QAC1E,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,6DAA6D,EAAE,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAY;QAC5B,MAAM,UAAU,GACd,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3F,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,OAAO,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC1C,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,WAAW,CAAC;QACzC,IAAI,CAAC,QAAQ,GAAG,iCAAiC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3E,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,OAAO;YACL,MAAM,EAAE;gBACN,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,4BAA4B,QAAQ,CAAC,WAAW,EAAE;gBAClG,UAAU,EAAE;gBACZ,SAAS,CAAC,IAAI,CAAC,KAAK,UAAU,EAAE,IAAI,CAAC;gBACrC,UAAU,EAAE;gBACZ,IAAI,CAAC,QAAQ;gBACb,mBAAmB;aACpB,CAAC,IAAI,CAAC,IAAI,CAAC;SACb,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAClC,MAAM,KAAK,GAAG,uBAAuB,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5D,OAAO,SAAS,CAAC,IAAI,EAAE,0BAA0B,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,OAAe;QAC1C,IAAI,IAAI,CAAC,YAAY,KAAK,EAAE,EAAE,CAAC;YAC7B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,iBAAiB,GAAG,UAAU,CAAC;gBACpC,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;gBACzB,OAAO;oBACL,MAAM,EAAE;wBACN,OAAO;wBACP,UAAU,EAAE;wBACZ,kBAAkB;wBAClB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;qBAC7E,CAAC,IAAI,CAAC,IAAI,CAAC;iBACb,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,KAAK,UAAU,EAAE,KAAK,IAAI,CAAC,kBAAkB,EAAE,EAAE,EAAE,CAAC;IACjF,CAAC;IAED,kBAAkB;QAChB,IAAI,CAAC,QAAQ;YACX,yCAAyC,IAAI,CAAC,YAAY,GAAG;gBAC7D,WAAW,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;QACvB,OAAO,GAAG,IAAI,CAAC,QAAQ,qBAAqB,CAAC;IAC/C,CAAC;IAED,gBAAgB,CAAC,MAAc;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC1C,MAAM,MAAM,GACV,IAAI,CAAC,iBAAiB,CAAC,KAAK,GAAG,CAAC,CAAC;YACjC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CACzB,CAAC,SAAS,EAAE,EAAE,CACZ,SAAS,CAAC,EAAE,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,WAAW,EAAE,CACnF,CAAC;QACJ,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,aAAa,CAAC,qCAAqC,EAAE;gBAC7D,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,0BAA0B;aACjC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC;IAC/C,CAAC;IAED,cAAc,CAAC,OAAe;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC;QACpB,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO;gBACL,MAAM,EAAE,WAAW,QAAQ,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK;gBACtE,OAAO,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,+BAA+B,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;IACpE,CAAC;CACF"}

package/dist/mcp/context.d.ts

@@ -0,0 +1,21 @@
+import type { ActionContext } from '../actions/context.js';
+import type { CredentialStore } from '../auth/credential-store.js';
+import type { AutoevalConfiguration } from '../config.js';
+import type { FetchImplementation } from '../api/client.js';
+import { type PollClock } from '../polling/clock.js';
+export type AutoevalMcpContext = {
+    actions: ActionContext;
+    configuration: AutoevalConfiguration;
+    clock: PollClock;
+    signal?: AbortSignal;
+};
+export type AutoevalMcpContextDependencies = {
+    configuration: AutoevalConfiguration;
+    environment?: Readonly<Record<string, string | undefined>>;
+    store?: CredentialStore;
+    fetchImplementation?: FetchImplementation;
+    clock?: PollClock;
+    signal?: AbortSignal;
+};
+export declare function createAutoevalMcpContext(dependencies: AutoevalMcpContextDependencies): Promise<AutoevalMcpContext>;
+//# sourceMappingURL=context.d.ts.map

package/dist/mcp/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/mcp/context.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAmB,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAGtE,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,aAAa,CAAC;IACvB,aAAa,EAAE,qBAAqB,CAAC;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,8BAA8B,GAAG;IAC3C,aAAa,EAAE,qBAAqB,CAAC;IACrC,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3D,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;IAC1C,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,wBAAsB,wBAAwB,CAC5C,YAAY,EAAE,8BAA8B,GAC3C,OAAO,CAAC,kBAAkB,CAAC,CAgB7B"}

package/dist/mcp/context.js

@@ -0,0 +1,19 @@
+import { SystemPollClock } from '../polling/clock.js';
+import { createAuthenticatedActionContext } from '../runtime/action-context.js';
+export async function createAutoevalMcpContext(dependencies) {
+    const actions = await createAuthenticatedActionContext({
+        configuration: dependencies.configuration,
+        ...(dependencies.environment ? { environment: dependencies.environment } : {}),
+        ...(dependencies.store ? { store: dependencies.store } : {}),
+        ...(dependencies.fetchImplementation
+            ? { fetchImplementation: dependencies.fetchImplementation }
+            : {}),
+    });
+    return {
+        actions,
+        configuration: dependencies.configuration,
+        clock: dependencies.clock ?? new SystemPollClock(),
+        ...(dependencies.signal ? { signal: dependencies.signal } : {}),
+    };
+}
+//# sourceMappingURL=context.js.map

package/dist/mcp/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/mcp/context.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAkB,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,gCAAgC,EAAE,MAAM,8BAA8B,CAAC;AAkBhF,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,YAA4C;IAE5C,MAAM,OAAO,GAAG,MAAM,gCAAgC,CAAC;QACrD,aAAa,EAAE,YAAY,CAAC,aAAa;QACzC,GAAG,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,GAAG,CAAC,YAAY,CAAC,mBAAmB;YAClC,CAAC,CAAC,EAAE,mBAAmB,EAAE,YAAY,CAAC,mBAAmB,EAAE;YAC3D,CAAC,CAAC,EAAE,CAAC;KACR,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,aAAa,EAAE,YAAY,CAAC,aAAa;QACzC,KAAK,EAAE,YAAY,CAAC,KAAK,IAAI,IAAI,eAAe,EAAE;QAClD,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC"}

package/dist/mcp/schemas.d.ts

@@ -0,0 +1,206 @@
+import { z } from 'zod';
+export declare const emptyInputSchema: z.ZodObject<{}, z.core.$strict>;
+export declare const listWorkspacesInputSchema: z.ZodObject<{
+    page: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const createWorkspaceInputSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const listEvaluationsInputSchema: z.ZodObject<{
+    workspaceId: z.ZodUUID;
+    page: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const createEvaluationInputSchema: z.ZodObject<{
+    workspaceId: z.ZodUUID;
+    evaluationName: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export declare const evaluationInputSchema: z.ZodObject<{
+    evaluationId: z.ZodUUID;
+}, z.core.$strict>;
+export declare const updateEvaluationTitleInputSchema: z.ZodObject<{
+    workspaceId: z.ZodUUID;
+    evaluationId: z.ZodUUID;
+    evaluationName: z.ZodString;
+    userSystemId: z.ZodString;
+    description: z.ZodOptional<z.ZodString>;
+    page: z.ZodOptional<z.ZodNumber>;
+    size: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const qualityStandardInputSchema: z.ZodObject<{
+    name: z.ZodString;
+    judge_model: z.ZodUUID;
+    rubric: z.ZodString;
+    anchors: z.ZodArray<z.ZodObject<{
+        input: z.ZodString;
+        response: z.ZodString;
+        reference: z.ZodString;
+        score: z.ZodUnion<readonly [z.ZodLiteral<1>, z.ZodLiteral<5>]>;
+        reasoning: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export declare const qualityStandardIdInputSchema: z.ZodObject<{
+    qualityStandardId: z.ZodUUID;
+}, z.core.$strict>;
+export declare const assignQualityStandardInputSchema: z.ZodObject<{
+    workspaceId: z.ZodUUID;
+    qualityStandardId: z.ZodUUID;
+}, z.core.$strict>;
+export declare const validateConfiguredEvaluationInputSchema: z.ZodObject<{
+    input: z.ZodObject<{
+        methodology: z.ZodObject<{
+            userSystemId: z.ZodOptional<z.ZodString>;
+            judgeModel: z.ZodString;
+            judgeModelId: z.ZodUUID;
+            runsPerScenario: z.ZodOptional<z.ZodNumber>;
+            evaluatorInstructions: z.ZodString;
+            changeLog: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>;
+        configuration: z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"scenario">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+            scenarioGeneration: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+        }, z.core.$strict>, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"conversation">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"agent_trace">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>], "contextType">, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodOptional<z.ZodString>;
+            primaryModelId: z.ZodUUID;
+            comparisonModelIds: z.ZodDefault<z.ZodArray<z.ZodUUID>>;
+            promptText: z.ZodString;
+            scenarios: z.ZodUnion<readonly [z.ZodArray<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>]>;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            expected: z.ZodOptional<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>]>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export declare const runConfiguredEvaluationInputSchema: z.ZodObject<{
+    evaluationId: z.ZodUUID;
+    input: z.ZodObject<{
+        methodology: z.ZodObject<{
+            userSystemId: z.ZodOptional<z.ZodString>;
+            judgeModel: z.ZodString;
+            judgeModelId: z.ZodUUID;
+            runsPerScenario: z.ZodOptional<z.ZodNumber>;
+            evaluatorInstructions: z.ZodString;
+            changeLog: z.ZodOptional<z.ZodString>;
+        }, z.core.$strict>;
+        configuration: z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"scenario">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+            scenarioGeneration: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+        }, z.core.$strict>, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"conversation">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodString;
+            contextType: z.ZodLiteral<"agent_trace">;
+            artifact: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
+            expected: z.ZodString;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>], "contextType">, z.ZodObject<{
+            evaluationName: z.ZodOptional<z.ZodString>;
+            contextName: z.ZodOptional<z.ZodString>;
+            primaryModelId: z.ZodUUID;
+            comparisonModelIds: z.ZodDefault<z.ZodArray<z.ZodUUID>>;
+            promptText: z.ZodString;
+            scenarios: z.ZodUnion<readonly [z.ZodArray<z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>>, z.ZodArray<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>]>;
+            referenceDocuments: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                filename: z.ZodString;
+                content: z.ZodString;
+            }, z.core.$strict>>>;
+            selectedMetrics: z.ZodArray<z.ZodString>;
+            expected: z.ZodOptional<z.ZodString>;
+            temperatureContext: z.ZodOptional<z.ZodNumber>;
+            autoStopEnabled: z.ZodOptional<z.ZodBoolean>;
+        }, z.core.$strict>]>;
+    }, z.core.$strict>;
+}, z.core.$strict>;
+export declare const runInputSchema: z.ZodObject<{
+    evaluationId: z.ZodUUID;
+}, z.core.$strict>;
+export declare const runReferenceInputSchema: z.ZodObject<{
+    evaluationId: z.ZodUUID;
+    runId: z.ZodUUID;
+}, z.core.$strict>;
+export declare const mcpToolOutputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    ok: z.ZodLiteral<true>;
+    data: z.ZodJSONSchema;
+}, z.core.$strict>, z.ZodObject<{
+    ok: z.ZodLiteral<false>;
+    error: z.ZodObject<{
+        kind: z.ZodString;
+        code: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strict>;
+}, z.core.$strict>], "ok">;
+//# sourceMappingURL=schemas.d.ts.map

package/dist/mcp/schemas.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../../src/mcp/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,eAAO,MAAM,gBAAgB,iCAAwB,CAAC;AAEtD,eAAO,MAAM,yBAAyB;;;kBAK3B,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;kBAM5B,CAAC;AAEZ,eAAO,MAAM,2BAA2B;;;kBAK7B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;kBAA4C,CAAC;AAE/E,eAAO,MAAM,gCAAgC;;;;;;;;kBAUlC,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;;kBAA8B,CAAC;AAEtE,eAAO,MAAM,4BAA4B;;kBAAiD,CAAC;AAE3F,eAAO,MAAM,gCAAgC;;;kBAKlC,CAAC;AAEZ,eAAO,MAAM,uCAAuC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAEzC,CAAC;AAEZ,eAAO,MAAM,kCAAkC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAKpC,CAAC;AAEZ,eAAO,MAAM,cAAc;;kBAA4C,CAAC;AAExE,eAAO,MAAM,uBAAuB;;;kBAKzB,CAAC;AAUZ,eAAO,MAAM,mBAAmB;;;;;;;;;;0BAG9B,CAAC"}

package/dist/mcp/schemas.js

@@ -0,0 +1,80 @@
+import { z } from 'zod';
+import { qualityStandardCreateSchema } from '../actions/quality-standards.js';
+import { configuredRunFileSchema } from '../configured-run/validation.js';
+const uuid = z.uuid();
+const page = z.number().int().positive().optional();
+const pageSize = z.number().int().min(1).max(100).optional();
+export const emptyInputSchema = z.object({}).strict();
+export const listWorkspacesInputSchema = z
+    .object({
+    page,
+    size: pageSize,
+})
+    .strict();
+export const createWorkspaceInputSchema = z
+    .object({
+    name: z.string().trim().min(1).max(120),
+    description: z.string().trim().min(1).max(1_000).optional(),
+})
+    .strict();
+export const listEvaluationsInputSchema = z
+    .object({
+    workspaceId: uuid,
+    page,
+    size: pageSize,
+})
+    .strict();
+export const createEvaluationInputSchema = z
+    .object({
+    workspaceId: uuid,
+    evaluationName: z.string().trim().min(1).max(120).optional(),
+})
+    .strict();
+export const evaluationInputSchema = z.object({ evaluationId: uuid }).strict();
+export const updateEvaluationTitleInputSchema = z
+    .object({
+    workspaceId: uuid,
+    evaluationId: uuid,
+    evaluationName: z.string().trim().min(1).max(120),
+    userSystemId: z.string().trim().min(1),
+    description: z.string().optional(),
+    page,
+    size: pageSize,
+})
+    .strict();
+export const qualityStandardInputSchema = qualityStandardCreateSchema;
+export const qualityStandardIdInputSchema = z.object({ qualityStandardId: uuid }).strict();
+export const assignQualityStandardInputSchema = z
+    .object({
+    workspaceId: uuid,
+    qualityStandardId: uuid,
+})
+    .strict();
+export const validateConfiguredEvaluationInputSchema = z
+    .object({ input: configuredRunFileSchema })
+    .strict();
+export const runConfiguredEvaluationInputSchema = z
+    .object({
+    evaluationId: uuid,
+    input: configuredRunFileSchema,
+})
+    .strict();
+export const runInputSchema = z.object({ evaluationId: uuid }).strict();
+export const runReferenceInputSchema = z
+    .object({
+    evaluationId: uuid,
+    runId: uuid,
+})
+    .strict();
+const mcpErrorSchema = z
+    .object({
+    kind: z.string(),
+    code: z.string(),
+    message: z.string(),
+})
+    .strict();
+export const mcpToolOutputSchema = z.discriminatedUnion('ok', [
+    z.object({ ok: z.literal(true), data: z.json() }).strict(),
+    z.object({ ok: z.literal(false), error: mcpErrorSchema }).strict(),
+]);
+//# sourceMappingURL=schemas.js.map

package/dist/mcp/schemas.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schemas.js","sourceRoot":"","sources":["../../src/mcp/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,2BAA2B,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACtB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC;AACpD,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;AAE7D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAEtD,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC;KACvC,MAAM,CAAC;IACN,IAAI;IACJ,IAAI,EAAE,QAAQ;CACf,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACvC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE;CAC5D,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,MAAM,CAAC;IACN,WAAW,EAAE,IAAI;IACjB,IAAI;IACJ,IAAI,EAAE,QAAQ;CACf,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC;KACzC,MAAM,CAAC;IACN,WAAW,EAAE,IAAI;IACjB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE;CAC7D,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAE/E,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC;KAC9C,MAAM,CAAC;IACN,WAAW,EAAE,IAAI;IACjB,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;IACjD,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,IAAI;IACJ,IAAI,EAAE,QAAQ;CACf,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,0BAA0B,GAAG,2BAA2B,CAAC;AAEtE,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAE3F,MAAM,CAAC,MAAM,gCAAgC,GAAG,CAAC;KAC9C,MAAM,CAAC;IACN,WAAW,EAAE,IAAI;IACjB,iBAAiB,EAAE,IAAI;CACxB,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,uCAAuC,GAAG,CAAC;KACrD,MAAM,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC;KAC1C,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,kCAAkC,GAAG,CAAC;KAChD,MAAM,CAAC;IACN,YAAY,EAAE,IAAI;IAClB,KAAK,EAAE,uBAAuB;CAC/B,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;AAExE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,YAAY,EAAE,IAAI;IAClB,KAAK,EAAE,IAAI;CACZ,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,cAAc,GAAG,CAAC;KACrB,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;CACpB,CAAC;KACD,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,kBAAkB,CAAC,IAAI,EAAE;IAC5D,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IAC1D,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE;CACnE,CAAC,CAAC"}

package/dist/mcp/server.d.ts

@@ -0,0 +1,5 @@
+import { McpServer } from '@modelcontextprotocol/server';
+import type { AutoevalMcpContext } from './context.js';
+import { type AutoevalMcpActions } from './tools.js';
+export declare function createAutoevalMcpServer(context: AutoevalMcpContext, actions?: AutoevalMcpActions): McpServer;
+//# sourceMappingURL=server.d.ts.map

package/dist/mcp/server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../src/mcp/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAEzD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,YAAY,CAAC;AAEpB,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,GAAE,kBAA0C,GAClD,SAAS,CAUX"}