@kradle/cli 0.0.17 → 0.2.0

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@kradle/cli",
-	"version": "0.0.17",
-	"description": "Kradle's CLI. Manage challenges, evaluations, agents and more!",
+	"version": "0.2.0",
+	"description": "Kradle's CLI. Manage challenges, experiments, agents and more!",
 	"keywords": [
 		"cli"
 	],
@@ -25,9 +25,11 @@
 		"watch": "rm -rf dist && tsc --watch",
 		"lint": "biome check .",
 		"format": "biome format --write . && biome check --write .",
-		"prepack": "sh scripts/prepack.sh",
-		"postpack": "sh scripts/postpack.sh",
-		"version": "oclif manifest && oclif readme && git add README.md"
+		"prepack": "npm run build && npm run version",
+		"version": "oclif manifest && oclif readme && git add README.md",
+		"test": "vitest run",
+		"test:watch": "vitest",
+		"test:integration": "vitest run --config vitest.config.ts"
 	},
 	"dependencies": {
 		"@google-cloud/storage": "^7.17.3",
@@ -55,7 +57,8 @@
 		"chai": "^4",
 		"oclif": "^4",
 		"tsx": "^4.20.6",
-		"typescript": "^5.9.3"
+		"typescript": "^5.9.3",
+		"vitest": "^2.1.9"
 	},
 	"engines": {
 		"node": ">=22.18.0"
@@ -76,8 +79,8 @@
 			"agent": {
 				"description": "Manage agents"
 			},
-			"evaluation": {
-				"description": "Manage and run evaluations"
+			"experiment": {
+				"description": "Manage and run experiments"
 			}
 		}
 	}

package/static/challenge.ts

@@ -1,8 +1,8 @@
+import path from 'node:path';
 import { Actions, createChallenge, DEFAULT_CHALLENGE_PATH } from "@kradle/challenges";
-import { config } from "./config.ts";
 
-// Extract challenge name from config
-const challenge_name = config.slug.split(":").at(-1)!;
+// Extract challenge name from folder name
+const challenge_name = path.basename(path.dirname(import.meta.url));
 const GAME_DURATION = 2 * 60 * 20; // 2 minutes
 
 createChallenge({
@@ -13,20 +13,19 @@ createChallenge({
 	custom_variables: {},
 })
 	.events(() => ({
-		start_challenge: {
-			actions: [Actions.announce({ message: ["Challenge starting!"] }), Actions.setTime({ time: "day" })],
+		start_challenge: () => {
+			Actions.announce({ message: ["Challenge starting!"] });
+			Actions.setTime({ time: "day" });
 		},
-		init_participants: {
-			actions: [
-				Actions.clear({ target: "all" }),
-				Actions.setAttribute({ attribute_: "minecraft:generic.max_health", value: 20, target: "all" }),
-			],
+		init_participants: () => {
+			Actions.clear({ target: "all" });
+			Actions.setAttribute({ attribute_: "minecraft:generic.max_health", value: 20, target: "all" });
 		},
-		end_challenge: {
-			actions: [Actions.announce({ message: ["Challenge ended!"] })],
+		end_challenge: () => {
+			Actions.announce({ message: ["Challenge ended!"] });
 		},
 	}))
 	.end_condition(({ alive_players }) => alive_players.equalTo(0))
 	.win_conditions(({ has_never_died }, { all }) => ({
 		[all]: has_never_died.equalTo(1),
-	}));
+	}));

package/static/experiment_template.ts

@@ -0,0 +1,114 @@
+/**
+ * Experiment template for running challenge benchmarks.
+ *
+ * This file defines an experiment manifest that specifies which agents to test
+ * against a challenge, how many runs to perform, and how many agents participate
+ * in each run.
+ *
+ * Usage:
+ *   1. Set CHALLENGE_SLUG to the challenge you want to run the experiment on
+ *   2. Modify AGENTS array to include the agents you want to test
+ *   3. Adjust NUM_RUNS and NUM_AGENTS_PER_RUN as needed
+ *   4. Optionally add tags in ADDITIONAL_TAGS for filtering results later
+ *
+ * You can also entirely change how runs are generated by modifying the `main` function.
+ *
+ * @returns A Manifest object containing all run configurations
+ */
+export function main(): Manifest {
+	// The challenge for this experiment (format: "username:challenge-name")
+	const CHALLENGE_SLUG = "[INSERT CHALLENGE SLUG HERE]";
+
+	// Pool of agents to sample from for each run
+	const AGENTS: string[] = [
+		"team-kradle:claude-sonnet-4",
+		"team-kradle:qwen3-coder",
+		"team-kradle:deepseek-chat-v3-1",
+		"team-kradle:grok-4",
+		"team-kradle:grok-code-fast-1",
+		"team-kradle:gpt-5",
+		"team-kradle:kimi-k2",
+		"team-kradle:gemini-2-5-flash",
+		"team-kradle:gemini-2-5-pro",
+		"team-kradle:glm-4-5-air",
+		"team-kradle:gpt-5-mini",
+		"team-kradle:o3-mini",
+		"team-kradle:codestral-2508",
+	];
+
+	// Total number of game runs to execute
+	const NUM_RUNS = 200;
+
+	// Number of agents randomly selected for each run
+	const NUM_AGENTS_PER_RUN = 4;
+
+	// Optional tags for categorizing/filtering experiment results
+	const ADDITIONAL_TAGS: string[] = [];
+
+	const runs: RunConfig[] = [];
+
+	for (let i = 0; i < NUM_RUNS; i++) {
+		// Randomly sample agents for the run
+		const selectedAgents = sampleWithoutReplacement(AGENTS, NUM_AGENTS_PER_RUN);
+
+		runs.push({
+			challenge_slug: CHALLENGE_SLUG,
+			participants: selectedAgents.map((agent) => ({ agent })),
+		});
+	}
+
+	return { runs, tags: ADDITIONAL_TAGS };
+}
+
+// ------------------------------------------------------------------------------------------------
+// Utility functions
+// ------------------------------------------------------------------------------------------------
+
+/**
+ * Randomly samples elements from an array without replacement.
+ *
+ * @param arr - The source array to sample from
+ * @param count - Number of elements to sample
+ * @returns Array of randomly selected elements
+ * @throws Error if count exceeds array length
+ */
+function sampleWithoutReplacement<T>(arr: T[], count: number): T[] {
+	if (count > arr.length) {
+		throw new Error("Sample size cannot be larger than array length.");
+	}
+
+	const copy = [...arr];
+	const result: T[] = [];
+
+	for (let i = 0; i < count; i++) {
+		const idx = Math.floor(Math.random() * copy.length);
+		result.push(copy[idx]);
+		copy.splice(idx, 1);
+	}
+
+	return result;
+}
+
+/** A participant in a challenge run */
+type Participant = {
+	/** Agent slug (format: "username:agent-name") */
+	agent: string;
+	/** Optional role assignment for role-based challenges */
+	role?: string;
+};
+
+/** Configuration for a single challenge run */
+type RunConfig = {
+	/** The challenge to run (format: "username:challenge-name") */
+	challenge_slug: string;
+	/** List of participants for this run */
+	participants: Participant[];
+};
+
+/** The experiment manifest returned by main() */
+type Manifest = {
+	/** Array of run configurations to execute */
+	runs: RunConfig[];
+	/** Optional tags for categorizing experiment results */
+	tags?: string[];
+};

package/static/project_template/dev.env

@@ -1,5 +1,5 @@
-WEB_API_URL=https://dev-api.kradle.ai/v0 #https://api.kradle.ai/v0
-WEB_URL=https://dev.kradle.ai #https:/.kradle.ai/workbench
-STUDIO_API_URL=http://localhost:2999/api/v0
-STUDIO_URL=kradle-dev://open #kradle://://open
-KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges-dev
+KRADLE_API_URL=https://dev-api.kradle.ai/v0 #https://api.kradle.ai/v0
+KRADLE_WEB_URL=https://dev.kradle.ai #https://kradle.ai
+KRADLE_STUDIO_API_URL=http://localhost:2999/api/v0
+KRADLE_STUDIO_URL=kradle-dev://open #kradle://open
+KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio-dev/challenges

package/static/project_template/prod.env

@@ -1,5 +1,5 @@
-WEB_API_URL=https://api.kradle.ai/v0 #https://dev-api.kradle.ai/v0
-WEB_URL=https://kradle.ai #https://dev.kradle.ai
-STUDIO_API_URL=http://localhost:2999/api/v0
-STUDIO_URL=kradle://open #kradle-dev://://open
+KRADLE_API_URL=https://api.kradle.ai/v0 #https://dev-api.kradle.ai/v0
+KRADLE_WEB_URL=https://kradle.ai #https://dev.kradle.ai
+KRADLE_STUDIO_API_URL=http://localhost:2999/api/v0
+KRADLE_STUDIO_URL=kradle://open #kradle-dev://open
 KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges

package/static/project_template/tsconfig.json

@@ -1,7 +1,7 @@
 {
 	"compilerOptions": {
 		"target": "ES2020",
-		"module": "CommonJS",
+		"module": "ES2020",
 		"outDir": "build",
 		"rootDir": ".",
 		"moduleResolution": "node",

package/dist/commands/challenge/multi-upload.d.ts

@@ -1,6 +0,0 @@
-import { Command } from "@oclif/core";
-export default class MultiUpload extends Command {
-    static description: string;
-    static examples: string[];
-    run(): Promise<void>;
-}

package/dist/commands/challenge/multi-upload.js

@@ -1,80 +0,0 @@
-import { Command } from "@oclif/core";
-import enquirer from "enquirer";
-import { Listr } from "listr2";
-import pc from "picocolors";
-import { ApiClient } from "../../lib/api-client.js";
-import { Challenge } from "../../lib/challenge.js";
-import { loadConfig } from "../../lib/config.js";
-export default class MultiUpload extends Command {
-    static description = "Interactively select and upload multiple challenges";
-    static examples = ["<%= config.bin %> <%= command.id %>"];
-    async run() {
-        // Not necessary since we don't have any args, but oclif will raise a warning if we don't parse the args
-        this.parse(MultiUpload);
-        const config = loadConfig();
-        const api = new ApiClient(config);
-        this.log(pc.blue(">> Loading challenges..."));
-        const [cloudChallenges, localChallenges, human] = await Promise.all([
-            api.listChallenges(),
-            Challenge.getLocalChallenges(),
-            api.getHuman(),
-        ]);
-        // Get local challenges that exist
-        const localChallengeIds = Object.keys(localChallenges);
-        if (localChallengeIds.length === 0) {
-            this.log(pc.yellow("No local challenges found"));
-            return;
-        }
-        // Create choices with status information
-        const cloudMap = new Map(cloudChallenges.map((c) => [c.slug, c]));
-        const choices = localChallengeIds.map((id) => {
-            const fullSlug = `${human.username}:${id}`;
-            const inCloud = cloudMap.has(fullSlug);
-            const status = inCloud ? pc.green("☁️ ") : pc.blue("💻");
-            return {
-                name: id,
-                message: `${status} ${id}`,
-            };
-        });
-        // Prompt user to select challenges
-        let response;
-        try {
-            response = await enquirer.prompt({
-                type: "multiselect",
-                name: "challenges",
-                message: "Select challenges to upload. ☁️ = exists in cloud, 📁 = exists locally only",
-                choices: choices.map((c) => c.message),
-            });
-        }
-        catch (error) {
-            this.log(pc.yellow(">> No challenges selected"));
-            return;
-        }
-        // Map back to challenge IDs
-        const selectedChallenges = response.challenges.map((selected) => {
-            const choice = choices.find((c) => c.message === selected);
-            return choice?.name || selected.replace(/^[✓⊡]\s+/, "");
-        });
-        // Create tasks for each challenge
-        const tasks = new Listr(selectedChallenges.map((challengeId) => ({
-            title: challengeId,
-            task: async () => {
-                const challenge = new Challenge(challengeId, config);
-                await challenge.build();
-                await challenge.upload(api);
-            },
-        })), {
-            concurrent: false,
-            exitOnError: false,
-        });
-        try {
-            await tasks.run();
-            this.log(pc.green(`\n✓ Uploaded ${selectedChallenges.length} challenges`));
-        }
-        catch (error) {
-            this.error(pc.red(`Some uploads failed: ${error instanceof Error ? error.message : String(error)}`), {
-                exit: false,
-            });
-        }
-    }
-}

package/dist/commands/evaluation/run.d.ts

@@ -1,13 +0,0 @@
-import { Command } from "@oclif/core";
-export default class Run extends Command {
-    static description: string;
-    static examples: string[];
-    static args: {
-        name: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
-    };
-    static flags: {
-        new: import("@oclif/core/interfaces").BooleanFlag<boolean>;
-        "max-concurrent": import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
-    };
-    run(): Promise<void>;
-}

package/dist/commands/evaluation/run.js

@@ -1,61 +0,0 @@
-import { Args, Command, Flags } from "@oclif/core";
-import pc from "picocolors";
-import { ApiClient } from "../../lib/api-client.js";
-import { loadConfig } from "../../lib/config.js";
-import { Evaluator } from "../../lib/evaluation/evaluator.js";
-const DEFAULT_MAX_CONCURRENT = 5;
-export default class Run extends Command {
-    static description = "Run an evaluation. If the evaluation had an ongoing iteration, it will resume from the last state.";
-    static examples = [
-        "<%= config.bin %> <%= command.id %> my-evaluation",
-        "<%= config.bin %> <%= command.id %> my-evaluation --new",
-        "<%= config.bin %> <%= command.id %> my-evaluation --max-concurrent 10",
-    ];
-    static args = {
-        name: Args.string({
-            description: "Name of the evaluation to run",
-            required: true,
-        }),
-    };
-    static flags = {
-        new: Flags.boolean({
-            char: "n",
-            description: "Start a new iteration of the evaluation",
-            default: false,
-        }),
-        "max-concurrent": Flags.integer({
-            char: "m",
-            description: "Maximum concurrent runs",
-            default: DEFAULT_MAX_CONCURRENT,
-        }),
-    };
-    async run() {
-        const { args, flags } = await this.parse(Run);
-        const config = loadConfig();
-        const api = new ApiClient(config);
-        const evaluator = new Evaluator(args.name, config, api);
-        // Check if evaluation exists
-        if (!(await evaluator.exists())) {
-            this.error(pc.red(`Evaluation '${args.name}' does not exist. Run 'kradle evaluation init ${args.name}' first.`));
-        }
-        // Check if config.ts exists
-        if (!(await evaluator.configExists())) {
-            this.error(pc.red(`Config file not found at ${evaluator.configPath}`));
-        }
-        this.log(pc.blue(`>> Starting evaluation: ${args.name}`));
-        if (flags.new) {
-            this.log(pc.yellow("   --new: Starting a new iteration of the evaluation"));
-        }
-        try {
-            await evaluator.run({
-                new: flags.new,
-                maxConcurrent: flags["max-concurrent"],
-                openMetabase: true,
-            });
-            this.log(pc.green("\n✓ Evaluation complete!"));
-        }
-        catch (error) {
-            this.error(pc.red(`Evaluation failed: ${error instanceof Error ? error.message : String(error)}`));
-        }
-    }
-}

package/dist/lib/config.d.ts

@@ -1,12 +0,0 @@
-import { z } from "zod";
-export declare const ConfigSchema: z.ZodObject<{
-    WEB_API_URL: z.ZodString;
-    WEB_URL: z.ZodString;
-    STUDIO_API_URL: z.ZodString;
-    STUDIO_URL: z.ZodString;
-    KRADLE_API_KEY: z.ZodString;
-    KRADLE_CHALLENGES_PATH: z.ZodDefault<z.ZodString>;
-    NAMESPACE: z.ZodDefault<z.ZodString>;
-}, z.core.$strip>;
-export type Config = z.infer<typeof ConfigSchema>;
-export declare function loadConfig(): Config;

package/dist/lib/config.js

@@ -1,49 +0,0 @@
-import os from "node:os";
-import path from "node:path";
-import { z } from "zod";
-import { untildify } from "./utils.js";
-export const ConfigSchema = z.object({
-    WEB_API_URL: z.string().url(),
-    WEB_URL: z.string().url(),
-    STUDIO_API_URL: z.string().url(),
-    STUDIO_URL: z.string(),
-    KRADLE_API_KEY: z.string(),
-    /**
-     * Absolute path to the challenges directory. Defaults to ~/Documents/kradle-studio/challenges.
-     */
-    KRADLE_CHALLENGES_PATH: z.string().default(path.join(os.homedir(), "Documents", "kradle-studio", "challenges")),
-    NAMESPACE: z.string().default("kradle"),
-});
-export function loadConfig() {
-    try {
-        /**
-         * First, resolve tildes in the KRADLE_CHALLENGES_PATH
-         */
-        let challengesPath;
-        if (process.env.KRADLE_CHALLENGES_PATH) {
-            challengesPath = untildify(process.env.KRADLE_CHALLENGES_PATH);
-        }
-        const config = ConfigSchema.parse({
-            WEB_API_URL: process.env.WEB_API_URL,
-            WEB_URL: process.env.WEB_URL,
-            STUDIO_API_URL: process.env.STUDIO_API_URL,
-            STUDIO_URL: process.env.STUDIO_URL,
-            KRADLE_API_KEY: process.env.KRADLE_API_KEY,
-            KRADLE_CHALLENGES_PATH: challengesPath,
-            NAMESPACE: process.env.NAMESPACE,
-        });
-        for (const field of Object.keys(config)) {
-            if (!config[field]) {
-                throw new Error(`Missing required config field: "${field}". Please check your .env file.`);
-            }
-        }
-        return config;
-    }
-    catch (error) {
-        if (error instanceof z.ZodError) {
-            const missingFields = error.issues.map((e) => e.path.join(".")).join(", ");
-            throw new Error(`Missing or invalid environment variables: ${missingFields}. Please check your .env file.`);
-        }
-        throw error;
-    }
-}

package/dist/lib/evaluation/evaluator.d.ts

@@ -1,88 +0,0 @@
-import type { ApiClient } from "../api-client.js";
-import type { Config } from "../config.js";
-import type { EvaluationMetadata, EvaluationOptions, Manifest, Progress } from "./types.js";
-export declare class Evaluator {
-    private name;
-    private config;
-    private api;
-    evaluationDir: string;
-    metadataPath: string;
-    private runner?;
-    private tui?;
-    private currentIteration?;
-    constructor(name: string, config: Config, api: ApiClient);
-    /**
-     * Get paths for a specific iteration
-     */
-    private getIterationPaths;
-    get configPath(): string;
-    /**
-     * Get the current iteration directory path
-     */
-    getCurrentIterationDir(): string;
-    /**
-     * Check if evaluation exists
-     */
-    exists(): Promise<boolean>;
-    /**
-     * Check if config.ts exists (master config)
-     */
-    configExists(): Promise<boolean>;
-    /**
-     * Load evaluation metadata
-     */
-    loadMetadata(): Promise<EvaluationMetadata | null>;
-    /**
-     * Save evaluation metadata
-     */
-    saveMetadata(metadata: EvaluationMetadata): Promise<void>;
-    /**
-     * Get the current iteration number, or -1 if none exists
-     */
-    getCurrentIterationNumber(): Promise<number>;
-    /**
-     * Create a new iteration
-     */
-    createNewIteration(): Promise<number>;
-    /**
-     * Get or create an iteration
-     * @param createNew - If true, always create a new iteration. Otherwise, use current iteration or create first one if none exists.
-     */
-    getOrCreateIteration(createNew: boolean): Promise<number>;
-    /**
-     * Load manifest from iteration
-     */
-    loadManifest(iteration: number): Promise<Manifest>;
-    /**
-     * Load progress from iteration
-     */
-    loadProgress(iteration: number): Promise<Progress | null>;
-    /**
-     * Save progress to current iteration
-     */
-    saveProgress(): Promise<void>;
-    /**
-     * Execute config.ts to generate manifest
-     */
-    generateManifest(configPath: string): Promise<Manifest>;
-    /**
-     * Execute config.ts file and return the manifest
-     */
-    private executeConfigFile;
-    /**
-     * Run the evaluation
-     */
-    run(options: EvaluationOptions): Promise<void>;
-    /**
-     * Handle state change from runner
-     */
-    private onRunStateChange;
-    /**
-     * Handle quit request
-     */
-    private handleQuit;
-    /**
-     * Open run in browser
-     */
-    private openRun;
-}