@kradle/cli 0.0.15 → 0.0.17

package/dist/lib/challenge.js

@@ -180,6 +180,9 @@ export class Challenge {
      */
     static async getLocalChallenges() {
         const challengesDir = path.resolve(process.cwd(), "challenges");
+        if (!existsSync(challengesDir)) {
+            return {};
+        }
         const entries = await readDirSorted(challengesDir);
         const challenges = {};
         for (const entry of entries) {

package/dist/lib/evaluation/evaluator.js

@@ -230,7 +230,7 @@ export class Evaluator {
             console.log("");
         }
         if (options.openMetabase ?? true) {
-            openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?tags=${iterationTag}`);
+            openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?run_tags=${iterationTag}`);
         }
         const errors = this.runner?.getAllStates().filter((state) => state.status === "error");
         if (errors?.length > 0) {

package/dist/lib/evaluation/runner.js

@@ -1,5 +1,5 @@
 const DEFAULT_MAX_CONCURRENT = 5;
-const RATE_LIMIT_BACKOFF_MS = 5000;
+const RATE_LIMIT_BACKOFF_MS = 15000;
 const STATUS_POLL_INTERVAL_MS = 2000;
 export class Runner {
     runs;
@@ -135,7 +135,13 @@ export class Runner {
     /**
      * Start a single run
      */
-    async startRun(index) {
+    async startRun(index, tries = 0) {
+        if (tries > 5) {
+            this.updateState(index, { status: "error", error: "Failed to start run after 3 tries" });
+            this.completedRuns.add(index);
+            this.activeRuns.delete(index);
+            return;
+        }
         const state = this.states[index];
         this.activeRuns.add(index);
         this.updateState(index, { status: "initializing", startTime: Date.now() });
@@ -165,6 +171,7 @@ export class Runner {
                 this.updateState(index, { status: "queued", error: undefined });
                 this.activeRuns.delete(index);
                 await this.delay(RATE_LIMIT_BACKOFF_MS);
+                await this.startRun(index, tries + 1);
                 return;
             }
             this.updateState(index, { status: "error", error: errorMessage });

package/oclif.manifest.json

@@ -401,5 +401,5 @@
       ]
     }
   },
-  "version": "0.0.15"
+  "version": "0.0.17"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@kradle/cli",
-	"version": "0.0.15",
+	"version": "0.0.17",
 	"description": "Kradle's CLI. Manage challenges, evaluations, agents and more!",
 	"keywords": [
 		"cli"

package/static/project_template/dev.env

@@ -2,4 +2,4 @@ WEB_API_URL=https://dev-api.kradle.ai/v0 #https://api.kradle.ai/v0
 WEB_URL=https://dev.kradle.ai #https:/.kradle.ai/workbench
 STUDIO_API_URL=http://localhost:2999/api/v0
 STUDIO_URL=kradle-dev://open #kradle://://open
-KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges
+KRADLE_CHALLENGES_PATH=~/Documents/kradle-studio/challenges-dev