@kradle/cli 0.0.5 → 0.0.7

package/README.md

@@ -2,7 +2,7 @@
 
 Kradle's CLI for managing Minecraft challenges, evaluations, agents, and more!
 
-## Kradle - private installation
+## Installation
 
 1. Install Kradle's CLI globally
 ```
@@ -48,7 +48,7 @@ KRADLE_API_KEY=your-api-key
 KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges
 ```
 
-## Commands
+## Challenge Commands
 
 ### Create Challenge
 
@@ -125,7 +125,7 @@ kradle challenge multi-upload
 
 Provides an interactive UI to select multiple challenges and uploads them in parallel.
 
-### Evaluations (beta)
+## Evaluations commands
 
 Plan and execute batches of runs across challenges/agents, with resumable iterations and a TUI.
 
@@ -148,6 +148,23 @@ Features:
 - Ink TUI: live status counts, elapsed times, scrollable run list; keys `q/Ctrl+C` quit, `↑/↓/j/k` move, `o` open run URL.
 - Per-iteration manifest: generated from the evaluation `config.ts` into `manifest.json` before runs start.
 
+## Publishing a New Version
+
+The CLI uses GitHub Actions for automated releases. To publish a new version:
+
+1. **Go to Actions** in the GitHub repository
+2. **Select "Create Release PR"** workflow from the sidebar
+3. **Click "Run workflow"** and choose the release type:
+   - `patch` - Bug fixes (0.0.5 → 0.0.6)
+   - `minor` - New features (0.0.5 → 0.1.0)
+   - `major` - Breaking changes (0.0.5 → 1.0.0)
+4. **Review and merge** the automatically created PR
+5. **Done!** The package is automatically published to npm when the PR is merged
+
+### Setup (one-time)
+
+For the publish workflow to work, we're using [NPM Trusted Publishers](https://docs.npmjs.com/trusted-publishers).
+
 ## Development
 
 ### Setup

package/dist/commands/evaluation/create.d.ts

@@ -0,0 +1,9 @@
+import { Command } from "@oclif/core";
+export default class Create extends Command {
+    static description: string;
+    static examples: string[];
+    static args: {
+        name: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    run(): Promise<void>;
+}

package/dist/commands/evaluation/create.js

@@ -0,0 +1,58 @@
+import { exec } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { Args, Command } from "@oclif/core";
+import pc from "picocolors";
+import { loadConfig } from "../../lib/config.js";
+import { getStaticResourcePath } from "../../lib/utils.js";
+export default class Create extends Command {
+    static description = "Create a new evaluation";
+    static examples = ["<%= config.bin %> <%= command.id %> my-evaluation"];
+    static args = {
+        name: Args.string({
+            description: "Name of the evaluation",
+            required: true,
+        }),
+    };
+    async run() {
+        const { args } = await this.parse(Create);
+        loadConfig(); // Validate config is available
+        const evaluationDir = path.resolve(process.cwd(), "evaluations", args.name);
+        const configPath = path.join(evaluationDir, "config.ts");
+        // Check if evaluation already exists
+        try {
+            await fs.access(evaluationDir);
+            this.error(pc.red(`Evaluation '${args.name}' already exists at ${evaluationDir}`));
+        }
+        catch {
+            // Directory doesn't exist, which is what we want
+        }
+        // Create evaluation directory
+        await fs.mkdir(evaluationDir, { recursive: true });
+        // Copy template
+        const templatePath = getStaticResourcePath("evaluation_template.ts");
+        await fs.copyFile(templatePath, configPath);
+        this.log(pc.green(`✓ Created evaluation '${args.name}'`));
+        this.log(pc.dim(`  Config: ${configPath}`));
+        // Offer to open in editor on macOS
+        if (process.platform === "darwin") {
+            this.log("");
+            this.log(pc.blue(">> Opening config.ts in your editor..."));
+            // Try Cursor first, then VS Code, then fallback to default
+            exec(`cursor "${configPath}" || code "${configPath}" || open "${configPath}"`, (error) => {
+                if (error) {
+                    this.log(pc.dim(`  Could not open editor automatically. Please open: ${configPath}`));
+                }
+            });
+        }
+        else {
+            this.log("");
+            this.log(pc.blue(`>> Edit the config file to define your runs:`));
+            this.log(pc.dim(`   ${configPath}`));
+        }
+        this.log("");
+        this.log(pc.blue(">> Next steps:"));
+        this.log(pc.dim(`   1. Edit ${path.basename(configPath)} to define your evaluation runs`));
+        this.log(pc.dim(`   2. Run: kradle evaluation run ${args.name}`));
+    }
+}

package/dist/commands/evaluation/init.js

@@ -52,7 +52,7 @@ export default class Init extends Command {
         }
         this.log("");
         this.log(pc.blue(">> Next steps:"));
-        this.log(pc.dim(`   1. Edit ${path.basename(configPath)} to define your evaluation runs`));
+        this.log(pc.dim(`   1. Edit ${path.basename(configPath)} to define your evaluation runs, and `));
         this.log(pc.dim(`   2. Run: kradle evaluation run ${args.name}`));
     }
 }

package/dist/commands/init.d.ts

@@ -4,7 +4,6 @@ export default class Init extends Command {
     static examples: string[];
     static flags: {
         name: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
-        dev: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         "api-key": import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
     };
     run(): Promise<void>;

package/dist/commands/init.js

@@ -14,11 +14,11 @@ export default class Init extends Command {
             description: "Project name",
             required: false,
         }),
-        dev: Flags.boolean({
-            char: "d",
-            description: "Use Kradle's development environment instead of production",
-            required: false,
-        }),
+        // dev: Flags.boolean({
+        // 	char: "d",
+        // 	description: "Use Kradle's development environment instead of production",
+        // 	required: false,
+        // }),
         "api-key": Flags.string({
             char: "k",
             description: "Kradle API key",
@@ -34,10 +34,10 @@ export default class Init extends Command {
             const nonHiddenFiles = files.filter((f) => !f.startsWith("."));
             const useCurrentDir = nonHiddenFiles.length === 0;
             if (useCurrentDir) {
-                this.log(pc.yellow("Current directory is empty, it will be used as the project directory."));
+                this.log(pc.yellow("Current directory is empty, it will be used to store challenges and evaluations."));
             }
             else {
-                this.log(pc.yellow("Current directory is not empty, a subdirectory will be created for the project."));
+                this.log(pc.yellow("Current directory is not empty, a subdirectory will be created to store challenges and evaluations."));
             }
             let projectName;
             if (flags.name) {
@@ -51,34 +51,36 @@ export default class Init extends Command {
                 const { name } = await enquirer.prompt({
                     type: "input",
                     name: "name",
-                    message: "Enter the project name:",
+                    message: "What should the directory be called?",
                     initial: initial,
                 });
                 projectName = name;
             }
-            let useDev = flags.dev;
-            if (!useDev) {
-                const { confirm } = await enquirer.prompt({
-                    type: "confirm",
-                    name: "confirm",
-                    message: "Do you want to use Kradle's development environment?",
-                    initial: false,
-                });
-                useDev = confirm;
-            }
-            if (useDev) {
-                this.log(pc.yellow("Using Kradle's development environment."));
-            }
-            else {
-                this.log(pc.green("Using Kradle's production environment."));
-            }
+            // let useDev = flags.dev;
+            // if (!useDev) {
+            // 	const { confirm } = await enquirer.prompt<{ confirm: boolean }>({
+            // 		type: "confirm",
+            // 		name: "confirm",
+            // 		message: "Do you want to use Kradle's development environment?",
+            // 		initial: false,
+            // 	});
+            // 	useDev = confirm;
+            // }
+            // if (useDev) {
+            // 	this.log(pc.yellow("Using Kradle's development environment."));
+            // } else {
+            // 	this.log(pc.green("Using Kradle's production environment."));
+            // }
+            this.log();
+            this.log(pc.yellow("Cloud Analytics are only available in the development environment for now. Development environment will be used."));
+            const useDev = true;
             const domain = useDev ? "dev.kradle.ai" : "kradle.ai";
             let apiKey;
             if (flags["api-key"]) {
                 apiKey = flags["api-key"];
             }
             else {
-                this.log(pc.dim(`\nGet your API key at: https://${domain}/settings#api-keys`));
+                this.log(pc.dim(`Get your API key at: https://${domain}/settings#api-keys`));
                 const { key } = await enquirer.prompt({
                     type: "password",
                     name: "key",

package/dist/lib/api-client.js

@@ -12,7 +12,13 @@ const DEFAULT_CHALLENGE_SCHEMA = {
     objective: {
         fieldName: "success_rate",
         direction: "maximize",
-    },
+    } /*
+    endStates: {
+        "red": "Red team only wins",
+        "blue": "Blue team only wins",
+        "both": "Both teams win",
+        "none": "No team wins",
+    },*/,
 };
 export class ApiClient {
     config;

package/dist/lib/evaluation/evaluator.js

@@ -1,7 +1,6 @@
-import { exec } from "node:child_process";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { executeNodeCommand } from "../utils.js";
+import { executeNodeCommand, openInBrowser } from "../utils.js";
 import { Runner } from "./runner.js";
 import { TUI } from "./tui.js";
 import { EvaluationMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
@@ -195,7 +194,9 @@ export class Evaluator {
         // Load manifest
         const manifest = await this.loadManifest(iteration);
         // We have 2 mandatory tags: "eval-<evaluation-name>" and "eval-<evaluation-name>-iteration-<iteration>"
-        const tags = [`eval-${this.name}`, `eval-${this.name}-iteration-${iteration}`, ...(manifest.tags ?? [])];
+        const evaluationTag = `eval-${this.name}`;
+        const iterationTag = `${evaluationTag}-iteration-${iteration}`;
+        const tags = [evaluationTag, iterationTag, ...(manifest.tags ?? [])];
         // Create runner
         this.runner = new Runner(manifest.runs, this.api, this.config.WEB_URL, {
             maxConcurrent: options.maxConcurrent,
@@ -232,6 +233,9 @@ export class Evaluator {
         if (errors?.length > 0) {
             throw new Error(`${errors.map((error) => error.error).join("\n\n")}`);
         }
+        if (options.openMetabase ?? true) {
+            openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis&tags=${iterationTag}`);
+        }
     }
     /**
      * Handle state change from runner
@@ -258,18 +262,7 @@ export class Evaluator {
     openRun(index) {
         const url = this.runner?.getRunUrl(index);
         if (url) {
-            const platform = process.platform;
-            let command;
-            if (platform === "darwin") {
-                command = `open "${url}"`;
-            }
-            else if (platform === "win32") {
-                command = `start "${url}"`;
-            }
-            else {
-                command = `xdg-open "${url}"`;
-            }
-            exec(command);
+            openInBrowser(url);
         }
     }
 }

package/dist/lib/evaluation/index.d.ts

@@ -1,4 +1,4 @@
-export * from "./types.js";
 export { Evaluator } from "./evaluator.js";
 export { Runner } from "./runner.js";
 export { TUI } from "./tui.js";
+export * from "./types.js";

package/dist/lib/evaluation/index.js

@@ -1,4 +1,4 @@
-export * from "./types.js";
 export { Evaluator } from "./evaluator.js";
 export { Runner } from "./runner.js";
 export { TUI } from "./tui.js";
+export * from "./types.js";

package/dist/lib/evaluation/tui.js

@@ -35,6 +35,7 @@ const RenderRunLine = ({ state, total, isSelected, padding, }) => {
     const agents = state.config.participants.map((p) => p.agent.split(":").pop() ?? p.agent).join(", ");
     const summary = `${state.config.challenge_slug} (${agents})`;
     const maxSummaryLength = getVisibleColumns() - indexLabel.length - statusLabel.length - (elapsedLabel ? elapsedLabel.length : 0) - 4; // 4 for the spaces and emoji
+    // biome-ignore lint/style/useTemplate: template literal would be less readable
     const summaryText = summary.length > maxSummaryLength ? summary.slice(0, maxSummaryLength - 1) + "…" : summary;
     return (_jsxs(Text, { inverse: isSelected, children: [_jsx(Text, { color: color, children: icon }), " ", indexLabel, " ", _jsx(Text, { color: color, children: statusLabel }), elapsedLabel ? (_jsxs(_Fragment, { children: [" ", _jsx(Text, { dimColor: true, children: elapsedLabel })] })) : null, " ", _jsx(Text, { dimColor: true, children: summaryText })] }));
 };

package/dist/lib/evaluation/types.d.ts

@@ -120,6 +120,7 @@ export type EvaluationMetadata = z.infer<typeof EvaluationMetadataSchema>;
 export interface EvaluationOptions {
     new: boolean;
     maxConcurrent: number;
+    openMetabase?: boolean;
 }
 export declare const STATUS_ICONS: Record<RunStatus, {
     icon: string;

package/dist/lib/schemas.d.ts

@@ -35,6 +35,7 @@ export declare const ChallengeSchema: z.ZodObject<{
             minimize: "minimize";
         }>;
     }, z.core.$strip>;
+    endStates: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
     creationTime: z.ZodOptional<z.ZodString>;
     updateTime: z.ZodOptional<z.ZodString>;
     creator: z.ZodOptional<z.ZodString>;
@@ -76,6 +77,7 @@ export declare const ChallengesResponseSchema: z.ZodObject<{
                 minimize: "minimize";
             }>;
         }, z.core.$strip>;
+        endStates: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         creationTime: z.ZodOptional<z.ZodString>;
         updateTime: z.ZodOptional<z.ZodString>;
         creator: z.ZodOptional<z.ZodString>;

package/dist/lib/schemas.js

@@ -23,6 +23,7 @@ export const ChallengeSchema = z.object({
         fieldName: z.string(),
         direction: z.enum(["maximize", "minimize"]),
     }),
+    endStates: z.record(z.string(), z.string()).optional(),
     creationTime: z.string().optional(),
     updateTime: z.string().optional(),
     creator: z.string().optional(),

package/dist/lib/utils.d.ts

@@ -87,3 +87,10 @@ export declare function executeCommand(command: string, args: string[], options?
  * @returns A promise that resolves with the stdout of the command.
  */
 export declare function executeNodeCommand(args: string[], config: Config): Promise<string>;
+/**
+ * Open a URL in the default browser.
+ * This is fire-and-forget, so we don't wait for it to complete.
+ *
+ * @param url The URL to open.
+ */
+export declare function openInBrowser(url: string): void;

package/dist/lib/utils.js

@@ -1,4 +1,4 @@
-import { fork, spawn } from "node:child_process";
+import { exec, fork, spawn } from "node:child_process";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
@@ -168,3 +168,23 @@ export async function executeCommand(command, args, options) {
 export async function executeNodeCommand(args, config) {
     return executeCommand(process.execPath, args, { env: config });
 }
+/**
+ * Open a URL in the default browser.
+ * This is fire-and-forget, so we don't wait for it to complete.
+ *
+ * @param url The URL to open.
+ */
+export function openInBrowser(url) {
+    const platform = process.platform;
+    let command;
+    if (platform === "darwin") {
+        command = `open "${url}"`;
+    }
+    else if (platform === "win32") {
+        command = `start "${url}"`;
+    }
+    else {
+        command = `xdg-open "${url}"`;
+    }
+    exec(command);
+}

package/oclif.manifest.json

@@ -17,14 +17,6 @@
           "multiple": false,
           "type": "option"
         },
-        "dev": {
-          "char": "d",
-          "description": "Use Kradle's development environment instead of production",
-          "name": "dev",
-          "required": false,
-          "allowNo": false,
-          "type": "boolean"
-        },
         "api-key": {
           "char": "k",
           "description": "Kradle API key",
@@ -305,6 +297,36 @@
         "watch.js"
       ]
     },
+    "evaluation:create": {
+      "aliases": [],
+      "args": {
+        "name": {
+          "description": "Name of the evaluation",
+          "name": "name",
+          "required": true
+        }
+      },
+      "description": "Create a new evaluation",
+      "examples": [
+        "<%= config.bin %> <%= command.id %> my-evaluation"
+      ],
+      "flags": {},
+      "hasDynamicHelp": false,
+      "hiddenAliases": [],
+      "id": "evaluation:create",
+      "pluginAlias": "@kradle/cli",
+      "pluginName": "@kradle/cli",
+      "pluginType": "core",
+      "strict": true,
+      "enableJsonFlag": false,
+      "isESM": true,
+      "relativePath": [
+        "dist",
+        "commands",
+        "evaluation",
+        "create.js"
+      ]
+    },
     "evaluation:init": {
       "aliases": [],
       "args": {
@@ -409,5 +431,5 @@
       ]
     }
   },
-  "version": "0.0.5"
+  "version": "0.0.7"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kradle/cli",
-	"version": "0.0.5",
+	"version": "0.0.7",
 	"description": "Kradle's CLI. Manage challenges, evaluations, agents and more!",
 	"keywords": [
 		"cli"
@@ -24,11 +24,10 @@
 		"build": "rm -rf dist && tsc",
 		"watch": "rm -rf dist && tsc --watch",
 		"lint": "biome check .",
-		"lint:fix": "biome check --write .",
-		"format": "biome format --write .",
+		"format": "biome format --write . && biome check --write .",
 		"prepack": "sh scripts/prepack.sh",
 		"postpack": "sh scripts/postpack.sh",
-		"version": "oclif readme && git add README.md"
+		"version": "oclif manifest && oclif readme && git add README.md"
 	},
 	"dependencies": {
 		"@google-cloud/storage": "^7.17.3",