@kradle/cli 0.0.17 → 0.2.0

package/dist/lib/evaluation/evaluator.js

@@ -1,268 +0,0 @@
-import fs from "node:fs/promises";
-import path from "node:path";
-import { executeNodeCommand, openInBrowser } from "../utils.js";
-import { Runner } from "./runner.js";
-import { TUI } from "./tui.js";
-import { EvaluationMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
-export class Evaluator {
-    name;
-    config;
-    api;
-    evaluationDir;
-    metadataPath;
-    runner;
-    tui;
-    currentIteration;
-    constructor(name, config, api) {
-        this.name = name;
-        this.config = config;
-        this.api = api;
-        this.evaluationDir = path.resolve(process.cwd(), "evaluations", name);
-        this.metadataPath = path.join(this.evaluationDir, ".evaluation.json");
-    }
-    /**
-     * Get paths for a specific iteration
-     */
-    getIterationPaths(iteration) {
-        const iterationDir = path.join(this.evaluationDir, "iterations", iteration.toString().padStart(3, "0"));
-        return {
-            iterationDir,
-            configPath: path.join(iterationDir, "config.ts"),
-            manifestPath: path.join(iterationDir, "manifest.json"),
-            progressPath: path.join(iterationDir, "progress.json"),
-        };
-    }
-    get configPath() {
-        return path.join(this.evaluationDir, "config.ts");
-    }
-    /**
-     * Get the current iteration directory path
-     */
-    getCurrentIterationDir() {
-        if (this.currentIteration === undefined) {
-            throw new Error("No iteration set");
-        }
-        return this.getIterationPaths(this.currentIteration).iterationDir;
-    }
-    /**
-     * Check if evaluation exists
-     */
-    async exists() {
-        try {
-            await fs.access(this.evaluationDir);
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /**
-     * Check if config.ts exists (master config)
-     */
-    async configExists() {
-        try {
-            await fs.access(this.configPath);
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /**
-     * Load evaluation metadata
-     */
-    async loadMetadata() {
-        try {
-            const content = await fs.readFile(this.metadataPath, "utf-8");
-            const data = JSON.parse(content);
-            return EvaluationMetadataSchema.parse(data);
-        }
-        catch {
-            return null;
-        }
-    }
-    /**
-     * Save evaluation metadata
-     */
-    async saveMetadata(metadata) {
-        await fs.writeFile(this.metadataPath, JSON.stringify(metadata, null, 2));
-    }
-    /**
-     * Get the current iteration number, or -1 if none exists
-     */
-    async getCurrentIterationNumber() {
-        const metadata = await this.loadMetadata();
-        return metadata?.currentIteration ?? -1;
-    }
-    /**
-     * Create a new iteration
-     */
-    async createNewIteration() {
-        const currentIteration = await this.getCurrentIterationNumber();
-        const newIteration = currentIteration + 1;
-        const paths = this.getIterationPaths(newIteration);
-        // Create iteration directory
-        await fs.mkdir(paths.iterationDir, { recursive: true });
-        // Copy master config to iteration
-        const masterConfigPath = path.join(this.evaluationDir, "config.ts");
-        await fs.copyFile(masterConfigPath, paths.configPath);
-        // Generate manifest from config
-        const manifest = await this.generateManifest(paths.configPath);
-        await fs.writeFile(paths.manifestPath, JSON.stringify(manifest, null, 2));
-        // Update metadata
-        await this.saveMetadata({ currentIteration: newIteration });
-        this.currentIteration = newIteration;
-        return newIteration;
-    }
-    /**
-     * Get or create an iteration
-     * @param createNew - If true, always create a new iteration. Otherwise, use current iteration or create first one if none exists.
-     */
-    async getOrCreateIteration(createNew) {
-        if (createNew) {
-            return await this.createNewIteration();
-        }
-        const currentIteration = await this.getCurrentIterationNumber();
-        if (currentIteration < 0) {
-            return await this.createNewIteration();
-        }
-        this.currentIteration = currentIteration;
-        return currentIteration;
-    }
-    /**
-     * Load manifest from iteration
-     */
-    async loadManifest(iteration) {
-        const paths = this.getIterationPaths(iteration);
-        const content = await fs.readFile(paths.manifestPath, "utf-8");
-        const data = JSON.parse(content);
-        return ManifestSchema.parse(data);
-    }
-    /**
-     * Load progress from iteration
-     */
-    async loadProgress(iteration) {
-        try {
-            const paths = this.getIterationPaths(iteration);
-            const content = await fs.readFile(paths.progressPath, "utf-8");
-            const data = JSON.parse(content);
-            return ProgressSchema.parse(data);
-        }
-        catch {
-            return null;
-        }
-    }
-    /**
-     * Save progress to current iteration
-     */
-    async saveProgress() {
-        if (!this.runner || this.currentIteration === undefined)
-            return;
-        const paths = this.getIterationPaths(this.currentIteration);
-        const progress = {
-            entries: this.runner.getProgressEntries(),
-            lastUpdated: Date.now(),
-        };
-        await fs.writeFile(paths.progressPath, JSON.stringify(progress, null, 2));
-    }
-    /**
-     * Execute config.ts to generate manifest
-     */
-    async generateManifest(configPath) {
-        const manifest = await this.executeConfigFile(configPath);
-        return ManifestSchema.parse(manifest);
-    }
-    /**
-     * Execute config.ts file and return the manifest
-     */
-    async executeConfigFile(configPath) {
-        // We spawn a new NodeJS process to execute & log the config file.
-        // We can't directly import the file because it would be cached, and import cache can't be invalidated.
-        const stdout = await executeNodeCommand([
-            "--experimental-transform-types",
-            "--no-warnings",
-            "-e",
-            `console.log(JSON.stringify(require("${configPath}").main()));`,
-        ], this.config);
-        return JSON.parse(stdout.trim());
-    }
-    /**
-     * Run the evaluation
-     */
-    async run(options) {
-        const iteration = await this.getOrCreateIteration(options.new);
-        // Load manifest
-        const manifest = await this.loadManifest(iteration);
-        // We have 2 mandatory tags: "eval-<evaluation-name>" and "eval-<evaluation-name>-iteration-<iteration>"
-        const evaluationTag = `eval-${this.name}`;
-        const iterationTag = `${evaluationTag}-iteration-${iteration}`;
-        const tags = [evaluationTag, iterationTag, ...(manifest.tags ?? [])];
-        // Create runner
-        this.runner = new Runner(manifest.runs, this.api, this.config.WEB_URL, {
-            maxConcurrent: options.maxConcurrent,
-            tags: tags,
-            onStateChange: () => this.onRunStateChange(),
-        });
-        // Restore progress if applicable
-        const progress = await this.loadProgress(iteration);
-        if (progress) {
-            this.runner.restoreProgress(progress.entries);
-        }
-        // Create TUI
-        this.tui = new TUI({
-            evaluationName: `${this.name} (iteration ${iteration})`,
-            onQuit: () => this.handleQuit(),
-            onOpenRun: (index) => this.openRun(index),
-        });
-        // Initial state update
-        this.tui.updateStates(this.runner.getAllStates());
-        this.tui.updateStatusCounts(this.runner.getStatusCounts());
-        // Start TUI
-        this.tui.start();
-        try {
-            // Execute runs
-            await this.runner.execute();
-            // Final save
-            await this.saveProgress();
-        }
-        finally {
-            this.tui.stop();
-            console.log("");
-        }
-        if (options.openMetabase ?? true) {
-            openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?run_tags=${iterationTag}`);
-        }
-        const errors = this.runner?.getAllStates().filter((state) => state.status === "error");
-        if (errors?.length > 0) {
-            throw new Error(`${errors.map((error) => error.error).join("\n\n")}`);
-        }
-    }
-    /**
-     * Handle state change from runner
-     */
-    onRunStateChange() {
-        if (this.tui && this.runner) {
-            this.tui.updateStates(this.runner.getAllStates());
-            this.tui.updateStatusCounts(this.runner.getStatusCounts());
-        }
-        // Periodically save progress
-        this.saveProgress().catch(() => { });
-    }
-    /**
-     * Handle quit request
-     */
-    handleQuit() {
-        this.runner?.stop();
-        this.tui?.stop();
-        process.exit(0);
-    }
-    /**
-     * Open run in browser
-     */
-    openRun(index) {
-        const url = this.runner?.getRunUrl(index);
-        if (url) {
-            openInBrowser(url);
-        }
-    }
-}

package/static/evaluation_template.ts

@@ -1,69 +0,0 @@
-export function main(): Manifest {
-	const CHALLENGE_SLUG = "[INSERT CHALLENGE SLUG HERE]";
-
-	const AGENTS: string[] = [
-		"team-kradle:claude-sonnet-4",
-		"team-kradle:qwen3-coder",
-		"team-kradle:deepseek-chat-v3-1",
-		"team-kradle:grok-4",
-		"team-kradle:grok-code-fast-1",
-		"team-kradle:gpt-5",
-		"team-kradle:kimi-k2",
-		"team-kradle:gemini-2-5-flash",
-		"team-kradle:gemini-2-5-pro",
-		"team-kradle:glm-4-5-air",
-		"team-kradle:gpt-5-mini",
-		"team-kradle:o3-mini",
-		"team-kradle:codestral-2508",
-	];
-
-	const NUM_RUNS = 200;
-	const NUM_AGENTS_PER_RUN = 4;
-
-	const ADDITIONAL_TAGS: string[] = [];
-
-	const runs: RunConfig[] = [];
-
-	for (let i = 0; i < NUM_RUNS; i++) {
-		const selectedAgents = sampleWithoutReplacement(AGENTS, NUM_AGENTS_PER_RUN);
-
-		runs.push({
-			challenge_slug: CHALLENGE_SLUG,
-			participants: selectedAgents.map((agent) => ({ agent })),
-		});
-	}
-
-	return { runs, tags: ADDITIONAL_TAGS };
-}
-
-function sampleWithoutReplacement<T>(arr: T[], count: number): T[] {
-	if (count > arr.length) {
-		throw new Error("Sample size cannot be larger than array length.");
-	}
-
-	const copy = [...arr];
-	const result: T[] = [];
-
-	for (let i = 0; i < count; i++) {
-		const idx = Math.floor(Math.random() * copy.length);
-		result.push(copy[idx]);
-		copy.splice(idx, 1);
-	}
-
-	return result;
-}
-
-type Participant = {
-	agent: string;
-	role?: string;
-};
-
-type RunConfig = {
-	challenge_slug: string;
-	participants: Participant[];
-};
-
-type Manifest = {
-	runs: RunConfig[];
-	tags?: string[];
-};