claude-test-bench 1.0.0

package/dist/server/services/workspace.d.ts

@@ -0,0 +1,11 @@
+import type { IWorkspaceBuilder, WorkspaceResult } from '../interfaces/workspace.js';
+import type { Scenario } from '../types/index.js';
+export declare class WorkspaceBuilder implements IWorkspaceBuilder {
+    createWorkspace(scenario: Scenario): Promise<WorkspaceResult>;
+    private writeClaudeMdFiles;
+    private writeRules;
+    private writeSkills;
+    private writeWorkspaceFiles;
+    private resolveContent;
+}
+//# sourceMappingURL=workspace.d.ts.map

package/dist/server/services/workspace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"workspace.d.ts","sourceRoot":"","sources":["../../../src/server/services/workspace.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AACrF,OAAO,KAAK,EAAE,QAAQ,EAAuD,MAAM,mBAAmB,CAAC;AAkCvG,qBAAa,gBAAiB,YAAW,iBAAiB;IAClD,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;YAyBrD,kBAAkB;YAmBlB,UAAU;YAkBV,WAAW;YAoBX,mBAAmB;YAcnB,cAAc;CAS7B"}

package/dist/server/services/workspace.js

@@ -0,0 +1,113 @@
+// ---------------------------------------------------------------------------
+// WorkspaceBuilder — creates isolated temp directories for each run
+// ---------------------------------------------------------------------------
+import os from 'node:os';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+// ---------------------------------------------------------------------------
+// Path validation
+// ---------------------------------------------------------------------------
+class PathValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PathValidationError';
+    }
+}
+/** Reject absolute paths and path traversal attempts. */
+function assertSafePath(value, label) {
+    if (path.isAbsolute(value)) {
+        throw new PathValidationError(`${label} must not be an absolute path: ${value}`);
+    }
+    const normalized = path.normalize(value);
+    if (normalized.startsWith('..') || normalized.includes(`${path.sep}..`)) {
+        throw new PathValidationError(`${label} must not contain '..': ${value}`);
+    }
+}
+function assertSafeName(value, label) {
+    if (value.includes('..') || value.includes(path.sep) || value.includes('/')) {
+        throw new PathValidationError(`${label} must be a simple name, got: ${value}`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+export class WorkspaceBuilder {
+    async createWorkspace(scenario) {
+        const workspacePath = path.join(os.tmpdir(), `ctb-run-${randomUUID()}`);
+        await fs.mkdir(workspacePath, { recursive: true });
+        try {
+            await this.writeClaudeMdFiles(workspacePath, scenario.claudeMdFiles);
+            await this.writeRules(workspacePath, scenario.rules);
+            await this.writeSkills(workspacePath, scenario.skills);
+            await this.writeWorkspaceFiles(workspacePath, scenario.workspaceFiles);
+        }
+        catch (err) {
+            // Cleanup on failure
+            await fs.rm(workspacePath, { recursive: true, force: true }).catch(() => { });
+            throw err;
+        }
+        return {
+            workspacePath,
+            cleanup: async () => {
+                await fs.rm(workspacePath, { recursive: true, force: true });
+            },
+        };
+    }
+    // ─── CLAUDE.md files ────────────────────────────────────────────────
+    async writeClaudeMdFiles(root, entries) {
+        for (const entry of entries) {
+            const content = await this.resolveContent(entry.content, entry.loadFromFile);
+            if (entry.role === 'project') {
+                await fs.writeFile(path.join(root, 'CLAUDE.md'), content, 'utf-8');
+            }
+            else {
+                const claudeDir = path.join(root, '.claude');
+                await fs.mkdir(claudeDir, { recursive: true });
+                await fs.writeFile(path.join(claudeDir, 'CLAUDE.md'), content, 'utf-8');
+            }
+        }
+    }
+    // ─── Rules ──────────────────────────────────────────────────────────
+    async writeRules(root, rules) {
+        if (rules.length === 0)
+            return;
+        const rulesDir = path.join(root, '.claude', 'rules');
+        await fs.mkdir(rulesDir, { recursive: true });
+        for (const rule of rules) {
+            assertSafeName(rule.name, 'rule name');
+            const content = await this.resolveContent(rule.content, rule.loadFromFile);
+            await fs.writeFile(path.join(rulesDir, `${rule.name}.md`), content, 'utf-8');
+        }
+    }
+    // ─── Skills ─────────────────────────────────────────────────────────
+    async writeSkills(root, skills) {
+        if (skills.length === 0)
+            return;
+        const skillsBase = path.join(root, '.claude', 'skills');
+        for (const skill of skills) {
+            assertSafeName(skill.name, 'skill name');
+            const skillDir = path.join(skillsBase, skill.name);
+            await fs.mkdir(skillDir, { recursive: true });
+            const content = await this.resolveContent(skill.content, skill.loadFromFile);
+            await fs.writeFile(path.join(skillDir, 'SKILL.md'), content, 'utf-8');
+        }
+    }
+    // ─── Workspace files ───────────────────────────────────────────────
+    async writeWorkspaceFiles(root, files) {
+        for (const file of files) {
+            assertSafePath(file.path, 'workspace file path');
+            const target = path.join(root, file.path);
+            await fs.mkdir(path.dirname(target), { recursive: true });
+            await fs.writeFile(target, file.content, 'utf-8');
+        }
+    }
+    // ─── Content resolution ────────────────────────────────────────────
+    async resolveContent(inlineContent, loadFromFile) {
+        if (loadFromFile) {
+            return fs.readFile(loadFromFile, 'utf-8');
+        }
+        return inlineContent;
+    }
+}
+//# sourceMappingURL=workspace.js.map

package/dist/server/services/workspace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"workspace.js","sourceRoot":"","sources":["../../../src/server/services/workspace.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,oEAAoE;AACpE,8EAA8E;AAE9E,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,mBAAoB,SAAQ,KAAK;IACrC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;IACpC,CAAC;CACF;AAED,yDAAyD;AACzD,SAAS,cAAc,CAAC,KAAa,EAAE,KAAa;IAClD,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,kCAAkC,KAAK,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,2BAA2B,KAAK,EAAE,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAa,EAAE,KAAa;IAClD,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,mBAAmB,CAAC,GAAG,KAAK,gCAAgC,KAAK,EAAE,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,OAAO,gBAAgB;IAC3B,KAAK,CAAC,eAAe,CAAC,QAAkB;QACtC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,UAAU,EAAE,EAAE,CAAC,CAAC;QACxE,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEnD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,kBAAkB,CAAC,aAAa,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAC;YACrE,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvD,MAAM,IAAI,CAAC,mBAAmB,CAAC,aAAa,EAAE,QAAQ,CAAC,cAAc,CAAC,CAAC;QACzE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,qBAAqB;YACrB,MAAM,EAAE,CAAC,EAAE,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC7E,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,OAAO;YACL,aAAa;YACb,OAAO,EAAE,KAAK,IAAI,EAAE;gBAClB,MAAM,EAAE,CAAC,EAAE,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC;SACF,CAAC;IACJ,CAAC;IAED,uEAAuE;IAE/D,KAAK,CAAC,kBAAkB,CAC9B,IAAY,EACZ,OAAiC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YAE7E,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC7B,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YACrE,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC7C,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC/C,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC;IACH,CAAC;IAED,uEAAuE;IAE/D,KAAK,CAAC,UAAU,CACtB,IAAY,EACZ,KAA2B;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YAC3E,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;IAED,uEAAuE;IAE/D,KAAK,CAAC,WAAW,CACvB,IAAY,EACZ,MAA6B;QAE7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QAExD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,cAAc,CAAC,KAAK,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE9C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YAC7E,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,sEAAsE;IAE9D,KAAK,CAAC,mBAAmB,CAC/B,IAAY,EACZ,KAA+B;QAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,sEAAsE;IAE9D,KAAK,CAAC,cAAc,CAC1B,aAAqB,EACrB,YAAqB;QAErB,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}

package/dist/server/types/evaluation.d.ts

@@ -0,0 +1,108 @@
+import type { ProviderConfig } from './provider.js';
+/** Configuration for an evaluator agent. */
+export interface EvaluatorConfig {
+    readonly provider: ProviderConfig;
+    readonly role: string;
+}
+/** Request to start an evaluation pipeline. */
+export interface EvaluationRequest {
+    readonly runId: string;
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly maxRounds: number;
+    readonly maxBudgetUsd?: number;
+}
+/** Lifecycle status of an evaluation. */
+export type EvaluationStatus = 'pending' | 'running' | 'completed' | 'failed';
+/** How well the answer matches the expected answer. */
+export interface AnswerComparison {
+    readonly matches: boolean;
+    readonly explanation: string;
+    readonly similarity: number;
+}
+/** Result for a single critical requirement. */
+export interface CriticalPartResult {
+    readonly requirement: string;
+    readonly met: boolean;
+    readonly evidence: string;
+}
+/** Per-dimension score from one evaluator. */
+export interface IndividualEvaluation {
+    readonly evaluatorRole: string;
+    readonly dimension: string;
+    readonly score: number;
+    readonly reasoning: string;
+}
+/** Did the agent follow the instructions in its CLAUDE.md / rules? */
+export interface InstructionCompliance {
+    readonly followed: readonly string[];
+    readonly violated: readonly string[];
+    readonly notApplicable: readonly string[];
+    readonly overallCompliance: number;
+}
+/** Report on whether defined skills were invoked. */
+export interface SkillUsageReport {
+    readonly skillName: string;
+    readonly invoked: boolean;
+    readonly invocationCount: number;
+    readonly contextRelevant: boolean;
+}
+/** Report on whether defined subagents were invoked. */
+export interface SubagentUsageReport {
+    readonly subagentName: string;
+    readonly invoked: boolean;
+    readonly invocationCount: number;
+    readonly contextRelevant: boolean;
+}
+/** Aggregated report on how well the setup itself performed. */
+export interface SetupComplianceReport {
+    readonly instructionCompliance: InstructionCompliance;
+    readonly skillUsage: readonly SkillUsageReport[];
+    readonly subagentUsage: readonly SubagentUsageReport[];
+}
+/** Ledger tracking cost/usage per evaluator. */
+export interface EvaluatorLedger {
+    readonly evaluatorRole: string;
+    readonly totalCostUsd: number;
+    readonly totalTokensIn: number;
+    readonly totalTokensOut: number;
+    readonly roundsParticipated: number;
+}
+/** A single round of evaluation (there may be multiple rounds for consensus). */
+export interface EvaluationRound {
+    readonly roundNumber: number;
+    readonly evaluations: readonly IndividualEvaluation[];
+    readonly consensusReached: boolean;
+    readonly timestamp: string;
+}
+/** Synthesised final scores across evaluators and rounds. */
+export interface EvaluationSynthesis {
+    readonly dimensionScores: Readonly<Record<string, number>>;
+    readonly weightedTotal: number;
+    readonly confidence: number;
+    readonly dissenting: readonly string[];
+}
+/** Overall report on how effective a setup is across evaluations. */
+export interface SetupEffectivenessReport {
+    readonly setupId: string;
+    readonly averageScore: number;
+    readonly scenarioBreakdown: Readonly<Record<string, number>>;
+    readonly strengths: readonly string[];
+    readonly weaknesses: readonly string[];
+}
+/** The full evaluation record persisted after the pipeline completes. */
+export interface Evaluation {
+    readonly id: string;
+    readonly runId: string;
+    readonly status: EvaluationStatus;
+    readonly evaluators: readonly EvaluatorConfig[];
+    readonly rounds: readonly EvaluationRound[];
+    readonly answerComparison: AnswerComparison;
+    readonly criticalResults: readonly CriticalPartResult[];
+    readonly setupCompliance: SetupComplianceReport;
+    readonly synthesis: EvaluationSynthesis;
+    readonly ledger: readonly EvaluatorLedger[];
+    readonly totalCostUsd: number;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=evaluation.d.ts.map

package/dist/server/types/evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation.d.ts","sourceRoot":"","sources":["../../../src/server/types/evaluation.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAEpD,4CAA4C;AAC5C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,yCAAyC;AACzC,MAAM,MAAM,gBAAgB,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE9E,uDAAuD;AACvD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,8CAA8C;AAC9C,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,sEAAsE;AACtE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACpC;AAED,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;CACnC;AAED,gEAAgE;AAChE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,qBAAqB,EAAE,qBAAqB,CAAC;IACtD,QAAQ,CAAC,UAAU,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACjD,QAAQ,CAAC,aAAa,EAAE,SAAS,mBAAmB,EAAE,CAAC;CACxD;AAED,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;CACrC;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,SAAS,oBAAoB,EAAE,CAAC;IACtD,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,6DAA6D;AAC7D,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC;AAED,qEAAqE;AACrE,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC;AAED,yEAAyE;AACzE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,eAAe,EAAE,CAAC;IAChD,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAC5C,QAAQ,CAAC,gBAAgB,EAAE,gBAAgB,CAAC;IAC5C,QAAQ,CAAC,eAAe,EAAE,SAAS,kBAAkB,EAAE,CAAC;IACxD,QAAQ,CAAC,eAAe,EAAE,qBAAqB,CAAC;IAChD,QAAQ,CAAC,SAAS,EAAE,mBAAmB,CAAC;IACxC,QAAQ,CAAC,MAAM,EAAE,SAAS,eAAe,EAAE,CAAC;IAC5C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/server/types/evaluation.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Evaluation Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=evaluation.js.map

package/dist/server/types/evaluation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluation.js","sourceRoot":"","sources":["../../../src/server/types/evaluation.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E"}

package/dist/server/types/index.d.ts

@@ -0,0 +1,5 @@
+export type { ApiProviderConfig, ClaudeMdEntry, EffortLevel, McpHttpConfig, McpServerConfig, McpServerEntry, McpSseConfig, McpStdioConfig, OAuthProviderConfig, PermissionMode, Provider, ProviderConfig, RuleEntry, ScoringDimension, SkillEntry, SubagentEntry, ThinkingConfig, } from './provider.js';
+export type { Scenario, ScenarioCategory, WorkspaceFile } from './scenario.js';
+export type { Run, RunStatus, SDKMessageRecord } from './run.js';
+export type { AnswerComparison, CriticalPartResult, Evaluation, EvaluationRequest, EvaluationRound, EvaluationStatus, EvaluationSynthesis, EvaluatorConfig, EvaluatorLedger, IndividualEvaluation, InstructionCompliance, SetupComplianceReport, SetupEffectivenessReport, SkillUsageReport, SubagentUsageReport, } from './evaluation.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/server/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/server/types/index.ts"],"names":[],"mappings":"AAIA,YAAY,EACV,iBAAiB,EACjB,aAAa,EACb,WAAW,EACX,aAAa,EACb,eAAe,EACf,cAAc,EACd,YAAY,EACZ,cAAc,EACd,mBAAmB,EACnB,cAAc,EACd,QAAQ,EACR,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,cAAc,GACf,MAAM,eAAe,CAAC;AAEvB,YAAY,EAAE,QAAQ,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE/E,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjE,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,mBAAmB,EACnB,eAAe,EACf,eAAe,EACf,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,mBAAmB,GACpB,MAAM,iBAAiB,CAAC"}

package/dist/server/types/index.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Central re-export for all shared types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=index.js.map

package/dist/server/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/types/index.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,yCAAyC;AACzC,8EAA8E"}

package/dist/server/types/provider.d.ts

@@ -0,0 +1,99 @@
+/** Discriminated union for provider authentication strategies. */
+export type ProviderConfig = ApiProviderConfig | OAuthProviderConfig;
+export interface ApiProviderConfig {
+    readonly kind: 'api';
+    readonly baseUrl: string;
+    readonly apiKey: string;
+    readonly model: string;
+}
+export interface OAuthProviderConfig {
+    readonly kind: 'oauth';
+    readonly oauthToken: string;
+    readonly model: string;
+}
+/** Discriminated union for thinking configuration. */
+export type ThinkingConfig = {
+    readonly kind: 'adaptive';
+} | {
+    readonly kind: 'enabled';
+    readonly budgetTokens: number;
+} | {
+    readonly kind: 'disabled';
+};
+/** A CLAUDE.md entry that can be inlined or loaded from a file. */
+export interface ClaudeMdEntry {
+    readonly role: 'project' | 'user';
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named rule entry. */
+export interface RuleEntry {
+    readonly name: string;
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named skill entry. */
+export interface SkillEntry {
+    readonly name: string;
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named subagent definition. */
+export interface SubagentEntry {
+    readonly name: string;
+    readonly description: string;
+    readonly prompt: string;
+    readonly tools?: readonly string[];
+    readonly disallowedTools?: readonly string[];
+    readonly model?: string;
+    readonly mcpServers?: readonly string[];
+    readonly skills?: readonly string[];
+    readonly maxTurns?: number;
+    readonly loadFromFile?: string;
+}
+/** MCP server transport configs — discriminated on `transport`. */
+export interface McpStdioConfig {
+    readonly transport: 'stdio';
+    readonly command: string;
+    readonly args?: readonly string[];
+    readonly env?: Readonly<Record<string, string>>;
+}
+export interface McpHttpConfig {
+    readonly transport: 'http';
+    readonly url: string;
+    readonly headers?: Readonly<Record<string, string>>;
+}
+export interface McpSseConfig {
+    readonly transport: 'sse';
+    readonly url: string;
+    readonly headers?: Readonly<Record<string, string>>;
+}
+export type McpServerConfig = McpStdioConfig | McpHttpConfig | McpSseConfig;
+/** A named MCP server entry. */
+export interface McpServerEntry {
+    readonly name: string;
+    readonly config: McpServerConfig;
+}
+/** Permission mode passed to the SDK. */
+export type PermissionMode = 'default' | 'acceptEdits' | 'bypassPermissions' | 'plan' | 'dontAsk';
+/** Effort level for Claude. 'none' = not applicable (e.g. non-Anthropic providers). */
+export type EffortLevel = 'none' | 'low' | 'medium' | 'high';
+/** Scoring dimension used to evaluate a run. */
+export interface ScoringDimension {
+    readonly name: string;
+    readonly weight: number;
+    readonly description: string;
+}
+/** A provider configuration that defines how to connect to an LLM provider. */
+export interface Provider {
+    readonly id: string;
+    readonly name: string;
+    readonly description: string;
+    readonly provider: ProviderConfig;
+    readonly thinking?: ThinkingConfig;
+    readonly effort?: EffortLevel;
+    readonly timeoutSeconds: number;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=provider.d.ts.map

package/dist/server/types/provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/server/types/provider.ts"],"names":[],"mappings":"AAIA,kEAAkE;AAClE,MAAM,MAAM,cAAc,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,sDAAsD;AACtD,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CAAC;AAElC,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,0BAA0B;AAC1B,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,2BAA2B;AAC3B,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,mEAAmE;AACnE,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACrD;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,aAAa,GAAG,YAAY,CAAC;AAE5E,gCAAgC;AAChC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;CAClC;AAED,yCAAyC;AACzC,MAAM,MAAM,cAAc,GACtB,SAAS,GACT,aAAa,GACb,mBAAmB,GACnB,MAAM,GACN,SAAS,CAAC;AAEd,uFAAuF;AACvF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE7D,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,+EAA+E;AAC/E,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/server/types/provider.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Provider Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=provider.js.map

package/dist/server/types/provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/server/types/provider.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E"}

package/dist/server/types/run.d.ts

@@ -0,0 +1,31 @@
+import type { Provider } from './provider.js';
+import type { Scenario } from './scenario.js';
+/** A raw SDK message captured during a run. */
+export interface SDKMessageRecord {
+    readonly timestamp: string;
+    readonly message: Readonly<Record<string, unknown>>;
+}
+/** Lifecycle status of a run. */
+export type RunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled';
+/** A single run: one provider + one scenario + captured output. */
+export interface Run {
+    readonly id: string;
+    readonly providerId: string;
+    readonly scenarioId: string;
+    readonly status: RunStatus;
+    readonly providerSnapshot: Provider;
+    readonly scenarioSnapshot: Scenario;
+    readonly messages: readonly SDKMessageRecord[];
+    readonly resultText: string;
+    readonly totalCostUsd: number;
+    readonly durationMs: number;
+    readonly numTurns: number;
+    readonly error?: string;
+    readonly reviewerProviderIds?: readonly string[];
+    readonly reviewerProviderSnapshots?: readonly Provider[];
+    readonly maxEvalRounds?: number;
+    readonly evaluationId?: string;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=run.d.ts.map

package/dist/server/types/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/server/types/run.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,+CAA+C;AAC/C,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;CACrD;AAED,iCAAiC;AACjC,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAErF,mEAAmE;AACnE,MAAM,WAAW,GAAG;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC/C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAExB,QAAQ,CAAC,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACjD,QAAQ,CAAC,yBAAyB,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACzD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/server/types/run.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Run Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=run.js.map

package/dist/server/types/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/server/types/run.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E"}

package/dist/server/types/scenario.d.ts

@@ -0,0 +1,32 @@
+import type { ClaudeMdEntry, McpServerEntry, PermissionMode, RuleEntry, ScoringDimension, SkillEntry, SubagentEntry } from './provider.js';
+/** Built-in category labels for scenarios (behavior/planning focused). */
+export type ScenarioCategory = 'planning' | 'instruction-following' | 'reasoning' | 'tool-strategy' | 'error-handling' | 'ambiguity-handling' | 'scope-management' | 'custom';
+/** A file that should exist in the workspace before the scenario runs. */
+export interface WorkspaceFile {
+    readonly path: string;
+    readonly content: string;
+}
+/** A scenario defines what to test, agent config, and how to grade. */
+export interface Scenario {
+    readonly id: string;
+    readonly name: string;
+    readonly category: ScenarioCategory;
+    readonly claudeMdFiles: readonly ClaudeMdEntry[];
+    readonly rules: readonly RuleEntry[];
+    readonly skills: readonly SkillEntry[];
+    readonly subagents: readonly SubagentEntry[];
+    readonly mcpServers: readonly McpServerEntry[];
+    readonly permissionMode: PermissionMode;
+    readonly maxTurns?: number;
+    readonly allowedTools?: readonly string[];
+    readonly disallowedTools?: readonly string[];
+    readonly prompt: string;
+    readonly workspaceFiles: readonly WorkspaceFile[];
+    readonly expectedAnswer: string;
+    readonly criticalRequirements: readonly string[];
+    readonly gradingGuidelines: string;
+    readonly scoringDimensions: readonly ScoringDimension[];
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=scenario.d.ts.map

package/dist/server/types/scenario.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.d.ts","sourceRoot":"","sources":["../../../src/server/types/scenario.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,aAAa,EACb,cAAc,EACd,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,UAAU,EACV,aAAa,EACd,MAAM,eAAe,CAAC;AAEvB,0EAA0E;AAC1E,MAAM,MAAM,gBAAgB,GACxB,UAAU,GACV,uBAAuB,GACvB,WAAW,GACX,eAAe,GACf,gBAAgB,GAChB,oBAAoB,GACpB,kBAAkB,GAClB,QAAQ,CAAC;AAEb,0EAA0E;AAC1E,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,uEAAuE;AACvE,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IAEpC,QAAQ,CAAC,aAAa,EAAE,SAAS,aAAa,EAAE,CAAC;IACjD,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAC;IACrC,QAAQ,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;IACvC,QAAQ,CAAC,SAAS,EAAE,SAAS,aAAa,EAAE,CAAC;IAC7C,QAAQ,CAAC,UAAU,EAAE,SAAS,cAAc,EAAE,CAAC;IAC/C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAE7C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAElD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,oBAAoB,EAAE,SAAS,MAAM,EAAE,CAAC;IACjD,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,gBAAgB,EAAE,CAAC;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/server/types/scenario.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Scenario Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=scenario.js.map

package/dist/server/types/scenario.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scenario.js","sourceRoot":"","sources":["../../../src/server/types/scenario.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E"}

package/dist/server/types/setup.d.ts

@@ -0,0 +1,99 @@
+/** Discriminated union for provider authentication strategies. */
+export type ProviderConfig = ApiProviderConfig | OAuthProviderConfig;
+export interface ApiProviderConfig {
+    readonly kind: 'api';
+    readonly baseUrl: string;
+    readonly apiKey: string;
+    readonly model: string;
+}
+export interface OAuthProviderConfig {
+    readonly kind: 'oauth';
+    readonly oauthToken: string;
+    readonly model: string;
+}
+/** Discriminated union for thinking configuration. */
+export type ThinkingConfig = {
+    readonly kind: 'adaptive';
+} | {
+    readonly kind: 'enabled';
+    readonly budgetTokens: number;
+} | {
+    readonly kind: 'disabled';
+};
+/** A CLAUDE.md entry that can be inlined or loaded from a file. */
+export interface ClaudeMdEntry {
+    readonly role: 'project' | 'user';
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named rule entry. */
+export interface RuleEntry {
+    readonly name: string;
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named skill entry. */
+export interface SkillEntry {
+    readonly name: string;
+    readonly content: string;
+    readonly loadFromFile?: string;
+}
+/** A named subagent definition. */
+export interface SubagentEntry {
+    readonly name: string;
+    readonly description: string;
+    readonly prompt: string;
+    readonly tools?: readonly string[];
+    readonly disallowedTools?: readonly string[];
+    readonly model?: string;
+    readonly mcpServers?: readonly string[];
+    readonly skills?: readonly string[];
+    readonly maxTurns?: number;
+    readonly loadFromFile?: string;
+}
+/** MCP server transport configs — discriminated on `transport`. */
+export interface McpStdioConfig {
+    readonly transport: 'stdio';
+    readonly command: string;
+    readonly args?: readonly string[];
+    readonly env?: Readonly<Record<string, string>>;
+}
+export interface McpHttpConfig {
+    readonly transport: 'http';
+    readonly url: string;
+    readonly headers?: Readonly<Record<string, string>>;
+}
+export interface McpSseConfig {
+    readonly transport: 'sse';
+    readonly url: string;
+    readonly headers?: Readonly<Record<string, string>>;
+}
+export type McpServerConfig = McpStdioConfig | McpHttpConfig | McpSseConfig;
+/** A named MCP server entry. */
+export interface McpServerEntry {
+    readonly name: string;
+    readonly config: McpServerConfig;
+}
+/** Permission mode passed to the SDK. */
+export type PermissionMode = 'default' | 'acceptEdits' | 'bypassPermissions' | 'plan' | 'dontAsk';
+/** Effort level for Claude. 'none' = not applicable (e.g. non-Anthropic providers). */
+export type EffortLevel = 'none' | 'low' | 'medium' | 'high';
+/** Scoring dimension used to evaluate a run. */
+export interface ScoringDimension {
+    readonly name: string;
+    readonly weight: number;
+    readonly description: string;
+}
+/** A complete test setup that defines how to connect to the provider. */
+export interface TestSetup {
+    readonly id: string;
+    readonly name: string;
+    readonly description: string;
+    readonly provider: ProviderConfig;
+    readonly thinking?: ThinkingConfig;
+    readonly effort?: EffortLevel;
+    readonly timeoutSeconds: number;
+    readonly createdAt: string;
+    readonly updatedAt: string;
+}
+//# sourceMappingURL=setup.d.ts.map

package/dist/server/types/setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../../src/server/types/setup.ts"],"names":[],"mappings":"AAIA,kEAAkE;AAClE,MAAM,MAAM,cAAc,GAAG,iBAAiB,GAAG,mBAAmB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,sDAAsD;AACtD,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GAC7B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CAAC;AAElC,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,0BAA0B;AAC1B,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,2BAA2B;AAC3B,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,QAAQ,CAAC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,mEAAmE;AACnE,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,GAAG,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACrD;AAED,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,aAAa,GAAG,YAAY,CAAC;AAE5E,gCAAgC;AAChC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;CAClC;AAED,yCAAyC;AACzC,MAAM,MAAM,cAAc,GACtB,SAAS,GACT,aAAa,GACb,mBAAmB,GACnB,MAAM,GACN,SAAS,CAAC;AAEd,uFAAuF;AACvF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE7D,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,yEAAyE;AACzE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAC;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B"}

package/dist/server/types/setup.js

@@ -0,0 +1,5 @@
+// ---------------------------------------------------------------------------
+// Provider & Setup Types
+// ---------------------------------------------------------------------------
+export {};
+//# sourceMappingURL=setup.js.map

package/dist/server/types/setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"setup.js","sourceRoot":"","sources":["../../../src/server/types/setup.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,yBAAyB;AACzB,8EAA8E"}

package/dist/src/server/index.d.ts

@@ -0,0 +1,13 @@
+import express from 'express';
+import type { IStorage } from './interfaces/storage.js';
+import type { ILogger } from './interfaces/logger.js';
+import type { IRunner } from './interfaces/runner.js';
+import type { IEvaluator } from './interfaces/evaluator.js';
+export interface AppDeps {
+    storage: IStorage;
+    logger: ILogger;
+    runner?: IRunner;
+    evaluator?: IEvaluator;
+}
+export declare function createApp(deps: AppDeps): express.Express;
+//# sourceMappingURL=index.d.ts.map

package/dist/src/server/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/server/index.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAK9B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAQ5D,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,QAAQ,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,UAAU,CAAC;CACxB;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAgExD"}