@kradle/cli 0.0.4 → 0.0.6

package/static/evaluation_template.ts

@@ -0,0 +1,69 @@
+export function main(): Manifest {
+	const CHALLENGE_SLUG = "[INSERT CHALLENGE SLUG HERE]";
+
+	const AGENTS: string[] = [
+		"team-kradle:claude-sonnet-4",
+		"team-kradle:qwen3-coder",
+		"team-kradle:deepseek-chat-v3-1",
+		"team-kradle:grok-4",
+		"team-kradle:grok-code-fast-1",
+		"team-kradle:gpt-5",
+		"team-kradle:kimi-k2",
+		"team-kradle:gemini-2-5-flash",
+		"team-kradle:gemini-2-5-pro",
+		"team-kradle:glm-4-5-air",
+		"team-kradle:gpt-5-mini",
+		"team-kradle:o3-mini",
+		"team-kradle:codestral-2508",
+	];
+
+	const NUM_RUNS = 200;
+	const NUM_AGENTS_PER_RUN = 4;
+
+	const ADDITIONAL_TAGS: string[] = [];
+
+	const runs: RunConfig[] = [];
+
+	for (let i = 0; i < NUM_RUNS; i++) {
+		const selectedAgents = sampleWithoutReplacement(AGENTS, NUM_AGENTS_PER_RUN);
+
+		runs.push({
+			challenge_slug: CHALLENGE_SLUG,
+			participants: selectedAgents.map((agent) => ({ agent })),
+		});
+	}
+
+	return { runs, tags: ADDITIONAL_TAGS };
+}
+
+function sampleWithoutReplacement<T>(arr: T[], count: number): T[] {
+	if (count > arr.length) {
+		throw new Error("Sample size cannot be larger than array length.");
+	}
+
+	const copy = [...arr];
+	const result: T[] = [];
+
+	for (let i = 0; i < count; i++) {
+		const idx = Math.floor(Math.random() * copy.length);
+		result.push(copy[idx]);
+		copy.splice(idx, 1);
+	}
+
+	return result;
+}
+
+type Participant = {
+	agent: string;
+	role?: string;
+};
+
+type RunConfig = {
+	challenge_slug: string;
+	participants: Participant[];
+};
+
+type Manifest = {
+	runs: RunConfig[];
+	tags?: string[];
+};

package/static/project_template/dev.env

@@ -2,5 +2,4 @@ WEB_API_URL=https://dev-api.kradle.ai/v0 #https://api.kradle.ai/v0
 WEB_URL=https://dev.kradle.ai #https:/.kradle.ai/workbench
 STUDIO_API_URL=http://localhost:2999/api/v0
 STUDIO_URL=kradle-dev://open #kradle://://open
-GCS_BUCKET=mckradle-3c267.firebasestorage.app #kradle-prod-storage
 KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges

package/static/project_template/prod.env

@@ -2,5 +2,4 @@ WEB_API_URL=https://api.kradle.ai/v0 #https://dev-api.kradle.ai/v0
 WEB_URL=https://kradle.ai #https://dev.kradle.ai
 STUDIO_API_URL=http://localhost:2999/api/v0
 STUDIO_URL=kradle://open #kradle-dev://://open
-GCS_BUCKET=kradle-prod-storage #mckradle-3c267.firebasestorage.app
 KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges