@kradle/cli 0.0.3 → 0.0.5

package/dist/lib/evaluation/evaluator.js

@@ -0,0 +1,275 @@
+import { exec } from "node:child_process";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { executeNodeCommand } from "../utils.js";
+import { Runner } from "./runner.js";
+import { TUI } from "./tui.js";
+import { EvaluationMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
+export class Evaluator {
+    name;
+    config;
+    api;
+    evaluationDir;
+    metadataPath;
+    runner;
+    tui;
+    currentIteration;
+    constructor(name, config, api) {
+        this.name = name;
+        this.config = config;
+        this.api = api;
+        this.evaluationDir = path.resolve(process.cwd(), "evaluations", name);
+        this.metadataPath = path.join(this.evaluationDir, ".evaluation.json");
+    }
+    /**
+     * Get paths for a specific iteration
+     */
+    getIterationPaths(iteration) {
+        const iterationDir = path.join(this.evaluationDir, "iterations", iteration.toString().padStart(3, "0"));
+        return {
+            iterationDir,
+            configPath: path.join(iterationDir, "config.ts"),
+            manifestPath: path.join(iterationDir, "manifest.json"),
+            progressPath: path.join(iterationDir, "progress.json"),
+        };
+    }
+    get configPath() {
+        return path.join(this.evaluationDir, "config.ts");
+    }
+    /**
+     * Get the current iteration directory path
+     */
+    getCurrentIterationDir() {
+        if (this.currentIteration === undefined) {
+            throw new Error("No iteration set");
+        }
+        return this.getIterationPaths(this.currentIteration).iterationDir;
+    }
+    /**
+     * Check if evaluation exists
+     */
+    async exists() {
+        try {
+            await fs.access(this.evaluationDir);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Check if config.ts exists (master config)
+     */
+    async configExists() {
+        try {
+            await fs.access(this.configPath);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Load evaluation metadata
+     */
+    async loadMetadata() {
+        try {
+            const content = await fs.readFile(this.metadataPath, "utf-8");
+            const data = JSON.parse(content);
+            return EvaluationMetadataSchema.parse(data);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Save evaluation metadata
+     */
+    async saveMetadata(metadata) {
+        await fs.writeFile(this.metadataPath, JSON.stringify(metadata, null, 2));
+    }
+    /**
+     * Get the current iteration number, or -1 if none exists
+     */
+    async getCurrentIterationNumber() {
+        const metadata = await this.loadMetadata();
+        return metadata?.currentIteration ?? -1;
+    }
+    /**
+     * Create a new iteration
+     */
+    async createNewIteration() {
+        const currentIteration = await this.getCurrentIterationNumber();
+        const newIteration = currentIteration + 1;
+        const paths = this.getIterationPaths(newIteration);
+        // Create iteration directory
+        await fs.mkdir(paths.iterationDir, { recursive: true });
+        // Copy master config to iteration
+        const masterConfigPath = path.join(this.evaluationDir, "config.ts");
+        await fs.copyFile(masterConfigPath, paths.configPath);
+        // Generate manifest from config
+        const manifest = await this.generateManifest(paths.configPath);
+        await fs.writeFile(paths.manifestPath, JSON.stringify(manifest, null, 2));
+        // Update metadata
+        await this.saveMetadata({ currentIteration: newIteration });
+        this.currentIteration = newIteration;
+        return newIteration;
+    }
+    /**
+     * Get or create an iteration
+     * @param createNew - If true, always create a new iteration. Otherwise, use current iteration or create first one if none exists.
+     */
+    async getOrCreateIteration(createNew) {
+        if (createNew) {
+            return await this.createNewIteration();
+        }
+        const currentIteration = await this.getCurrentIterationNumber();
+        if (currentIteration < 0) {
+            return await this.createNewIteration();
+        }
+        this.currentIteration = currentIteration;
+        return currentIteration;
+    }
+    /**
+     * Load manifest from iteration
+     */
+    async loadManifest(iteration) {
+        const paths = this.getIterationPaths(iteration);
+        const content = await fs.readFile(paths.manifestPath, "utf-8");
+        const data = JSON.parse(content);
+        return ManifestSchema.parse(data);
+    }
+    /**
+     * Load progress from iteration
+     */
+    async loadProgress(iteration) {
+        try {
+            const paths = this.getIterationPaths(iteration);
+            const content = await fs.readFile(paths.progressPath, "utf-8");
+            const data = JSON.parse(content);
+            return ProgressSchema.parse(data);
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Save progress to current iteration
+     */
+    async saveProgress() {
+        if (!this.runner || this.currentIteration === undefined)
+            return;
+        const paths = this.getIterationPaths(this.currentIteration);
+        const progress = {
+            entries: this.runner.getProgressEntries(),
+            lastUpdated: Date.now(),
+        };
+        await fs.writeFile(paths.progressPath, JSON.stringify(progress, null, 2));
+    }
+    /**
+     * Execute config.ts to generate manifest
+     */
+    async generateManifest(configPath) {
+        const manifest = await this.executeConfigFile(configPath);
+        return ManifestSchema.parse(manifest);
+    }
+    /**
+     * Execute config.ts file and return the manifest
+     */
+    async executeConfigFile(configPath) {
+        // We spawn a new NodeJS process to execute & log the config file.
+        // We can't directly import the file because it would be cached, and import cache can't be invalidated.
+        const stdout = await executeNodeCommand([
+            "--experimental-transform-types",
+            "--no-warnings",
+            "-e",
+            `console.log(JSON.stringify(require("${configPath}").main()));`,
+        ], this.config);
+        return JSON.parse(stdout.trim());
+    }
+    /**
+     * Run the evaluation
+     */
+    async run(options) {
+        const iteration = await this.getOrCreateIteration(options.new);
+        // Load manifest
+        const manifest = await this.loadManifest(iteration);
+        // We have 2 mandatory tags: "eval-<evaluation-name>" and "eval-<evaluation-name>-iteration-<iteration>"
+        const tags = [`eval-${this.name}`, `eval-${this.name}-iteration-${iteration}`, ...(manifest.tags ?? [])];
+        // Create runner
+        this.runner = new Runner(manifest.runs, this.api, this.config.WEB_URL, {
+            maxConcurrent: options.maxConcurrent,
+            tags: tags,
+            onStateChange: () => this.onRunStateChange(),
+        });
+        // Restore progress if applicable
+        const progress = await this.loadProgress(iteration);
+        if (progress) {
+            this.runner.restoreProgress(progress.entries);
+        }
+        // Create TUI
+        this.tui = new TUI({
+            evaluationName: `${this.name} (iteration ${iteration})`,
+            onQuit: () => this.handleQuit(),
+            onOpenRun: (index) => this.openRun(index),
+        });
+        // Initial state update
+        this.tui.updateStates(this.runner.getAllStates());
+        this.tui.updateStatusCounts(this.runner.getStatusCounts());
+        // Start TUI
+        this.tui.start();
+        try {
+            // Execute runs
+            await this.runner.execute();
+            // Final save
+            await this.saveProgress();
+        }
+        finally {
+            this.tui.stop();
+            console.log("");
+        }
+        const errors = this.runner?.getAllStates().filter((state) => state.status === "error");
+        if (errors?.length > 0) {
+            throw new Error(`${errors.map((error) => error.error).join("\n\n")}`);
+        }
+    }
+    /**
+     * Handle state change from runner
+     */
+    onRunStateChange() {
+        if (this.tui && this.runner) {
+            this.tui.updateStates(this.runner.getAllStates());
+            this.tui.updateStatusCounts(this.runner.getStatusCounts());
+        }
+        // Periodically save progress
+        this.saveProgress().catch(() => { });
+    }
+    /**
+     * Handle quit request
+     */
+    handleQuit() {
+        this.runner?.stop();
+        this.tui?.stop();
+        process.exit(0);
+    }
+    /**
+     * Open run in browser
+     */
+    openRun(index) {
+        const url = this.runner?.getRunUrl(index);
+        if (url) {
+            const platform = process.platform;
+            let command;
+            if (platform === "darwin") {
+                command = `open "${url}"`;
+            }
+            else if (platform === "win32") {
+                command = `start "${url}"`;
+            }
+            else {
+                command = `xdg-open "${url}"`;
+            }
+            exec(command);
+        }
+    }
+}

package/dist/lib/evaluation/index.d.ts

@@ -0,0 +1,4 @@
+export * from "./types.js";
+export { Evaluator } from "./evaluator.js";
+export { Runner } from "./runner.js";
+export { TUI } from "./tui.js";

package/dist/lib/evaluation/index.js

@@ -0,0 +1,4 @@
+export * from "./types.js";
+export { Evaluator } from "./evaluator.js";
+export { Runner } from "./runner.js";
+export { TUI } from "./tui.js";

package/dist/lib/evaluation/runner.d.ts

@@ -0,0 +1,80 @@
+import type { ApiClient } from "../api-client.js";
+import type { ProgressEntry, RunConfig, RunState, StatusCounts } from "./types.js";
+export declare class Runner {
+    private runs;
+    private api;
+    private baseUrl;
+    private states;
+    private activeRuns;
+    private completedRuns;
+    private nextIndex;
+    private stopped;
+    private maxConcurrent;
+    private tags;
+    private onStateChange?;
+    constructor(runs: RunConfig[], api: ApiClient, baseUrl: string, options?: {
+        maxConcurrent?: number;
+        tags?: string[];
+        onStateChange?: (index: number, state: RunState) => void;
+    });
+    /**
+     * Restore progress from a previous run
+     */
+    restoreProgress(entries: ProgressEntry[]): void;
+    /**
+     * Resume polling for runs that were in-flight before a crash/restart
+     */
+    private resumeInFlightRuns;
+    /**
+     * Get current run state by index
+     */
+    getRunState(index: number): RunState | undefined;
+    /**
+     * Get all run states
+     */
+    getAllStates(): RunState[];
+    /**
+     * Get status counts for TUI footer
+     */
+    getStatusCounts(): StatusCounts;
+    /**
+     * Update state and notify listener
+     */
+    private updateState;
+    /**
+     * Check if all runs are complete
+     */
+    isComplete(): boolean;
+    /**
+     * Stop scheduling new runs
+     */
+    stop(): void;
+    /**
+     * Start a single run
+     */
+    private startRun;
+    /**
+     * Poll run status until complete
+     */
+    private pollRunStatus;
+    /**
+     * Normalize API status to RunStatus
+     */
+    private normalizeStatus;
+    /**
+     * Main execution loop
+     */
+    execute(): Promise<void>;
+    /**
+     * Helper for delays
+     */
+    private delay;
+    /**
+     * Get progress entries for saving
+     */
+    getProgressEntries(): ProgressEntry[];
+    /**
+     * Get URL for a run
+     */
+    getRunUrl(index: number): string | undefined;
+}

package/dist/lib/evaluation/runner.js

@@ -0,0 +1,280 @@
+const DEFAULT_MAX_CONCURRENT = 5;
+const RATE_LIMIT_BACKOFF_MS = 5000;
+const STATUS_POLL_INTERVAL_MS = 2000;
+export class Runner {
+    runs;
+    api;
+    baseUrl;
+    states = [];
+    activeRuns = new Set();
+    completedRuns = new Set();
+    nextIndex = 0;
+    stopped = false;
+    maxConcurrent;
+    tags;
+    onStateChange;
+    constructor(runs, api, baseUrl, options = {}) {
+        this.runs = runs;
+        this.api = api;
+        this.baseUrl = baseUrl;
+        this.maxConcurrent = options.maxConcurrent ?? DEFAULT_MAX_CONCURRENT;
+        this.tags = options.tags ?? [];
+        // Validate tags respect regex
+        for (const tag of this.tags) {
+            if (!/^[a-zA-Z0-9()][a-zA-Z0-9()-]{1,34}[a-zA-Z0-9()]$/.test(tag)) {
+                throw new Error(`Invalid tag: ${tag}. Tags must start and end with a letter or number, and can only contain letters, numbers, hyphens, underscores, and parentheses.`);
+            }
+        }
+        this.onStateChange = options.onStateChange;
+        // Initialize all run states as queued
+        this.states = runs.map((config, index) => ({
+            index,
+            config,
+            status: "queued",
+        }));
+    }
+    /**
+     * Restore progress from a previous run
+     */
+    restoreProgress(entries) {
+        for (const entry of entries) {
+            if (entry.index < this.states.length) {
+                const state = this.states[entry.index];
+                state.status = entry.status;
+                state.runId = entry.runId;
+                state.startTime = entry.startTime;
+                if (entry.status === "completed" || entry.status === "finished" || entry.status === "game_over") {
+                    this.completedRuns.add(entry.index);
+                }
+                else if (entry.status === "error") {
+                    this.completedRuns.add(entry.index);
+                    state.error = entry.error;
+                }
+            }
+        }
+        // Find next index to process (first queued, non-completed)
+        const nextQueued = this.states.findIndex((state) => !this.completedRuns.has(state.index) && state.status === "queued");
+        this.nextIndex = nextQueued === -1 ? this.runs.length : nextQueued;
+    }
+    /**
+     * Resume polling for runs that were in-flight before a crash/restart
+     */
+    resumeInFlightRuns() {
+        for (const state of this.states) {
+            if (this.completedRuns.has(state.index))
+                continue;
+            if (state.status === "error")
+                continue;
+            if (!state.runId)
+                continue;
+            if (state.status === "queued")
+                continue;
+            if (this.activeRuns.has(state.index))
+                continue;
+            this.activeRuns.add(state.index);
+            // Resume polling the existing run without creating a new one
+            this.pollRunStatus(state.index, state.runId).catch(() => { });
+        }
+    }
+    /**
+     * Get current run state by index
+     */
+    getRunState(index) {
+        return this.states[index];
+    }
+    /**
+     * Get all run states
+     */
+    getAllStates() {
+        return [...this.states];
+    }
+    /**
+     * Get status counts for TUI footer
+     */
+    getStatusCounts() {
+        let completed = 0;
+        let active = 0;
+        let queued = 0;
+        let errors = 0;
+        for (const state of this.states) {
+            if (state.status === "completed" || state.status === "finished" || state.status === "game_over") {
+                completed++;
+            }
+            else if (state.status === "error") {
+                errors++;
+            }
+            else if (state.status === "queued") {
+                queued++;
+            }
+            else {
+                active++;
+            }
+        }
+        return { completed, active, queued, errors };
+    }
+    /**
+     * Update state and notify listener
+     */
+    updateState(index, updates) {
+        const state = this.states[index];
+        Object.assign(state, updates);
+        this.onStateChange?.(index, state);
+    }
+    /**
+     * Check if all runs are complete
+     */
+    isComplete() {
+        return this.completedRuns.size === this.runs.length;
+    }
+    /**
+     * Stop scheduling new runs
+     */
+    stop() {
+        this.stopped = true;
+    }
+    /**
+     * Start a single run
+     */
+    async startRun(index) {
+        const state = this.states[index];
+        this.activeRuns.add(index);
+        this.updateState(index, { status: "initializing", startTime: Date.now() });
+        try {
+            // Create the run via API
+            const response = await this.api.runChallenge({
+                challenge: state.config.challenge_slug,
+                participants: state.config.participants,
+            });
+            if (!response.runIds || response.runIds.length === 0) {
+                throw new Error("No run ID returned from API");
+            }
+            const runId = response.runIds[0];
+            this.updateState(index, { runId, status: "running" });
+            // Tag the run with all configured tags
+            await Promise.all(this.tags.map((tag) => this.api.tagRun(runId, tag)));
+            // Poll for completion
+            await this.pollRunStatus(index, runId);
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            // Check for rate limiting
+            if (errorMessage.includes("429") || errorMessage.toLowerCase().includes("rate limit")) {
+                // Re-queue for later
+                this.updateState(index, { status: "queued", error: undefined });
+                this.activeRuns.delete(index);
+                await this.delay(RATE_LIMIT_BACKOFF_MS);
+                return;
+            }
+            this.updateState(index, { status: "error", error: errorMessage });
+            this.completedRuns.add(index);
+            this.activeRuns.delete(index);
+        }
+    }
+    /**
+     * Poll run status until complete
+     */
+    async pollRunStatus(index, runId) {
+        while (!this.stopped) {
+            try {
+                const status = await this.api.getRunStatus(runId);
+                const normalizedStatus = this.normalizeStatus(status.status);
+                this.updateState(index, { status: normalizedStatus });
+                if (normalizedStatus === "completed" || normalizedStatus === "finished" || normalizedStatus === "game_over") {
+                    this.completedRuns.add(index);
+                    this.activeRuns.delete(index);
+                    return;
+                }
+                if (normalizedStatus === "error") {
+                    this.completedRuns.add(index);
+                    this.activeRuns.delete(index);
+                    return;
+                }
+                await this.delay(STATUS_POLL_INTERVAL_MS);
+            }
+            catch (error) {
+                // Network error, continue polling
+                await this.delay(STATUS_POLL_INTERVAL_MS * 2);
+            }
+        }
+        this.activeRuns.delete(index);
+    }
+    /**
+     * Normalize API status to RunStatus
+     */
+    normalizeStatus(apiStatus) {
+        const statusMap = {
+            queued: "queued",
+            pending: "queued",
+            initializing: "initializing",
+            watcher_connected: "watcher_connected",
+            participants_connected: "participants_connected",
+            started: "started",
+            running: "running",
+            recovering: "recovering",
+            completed: "completed",
+            game_over: "game_over",
+            finished: "finished",
+            error: "error",
+            failed: "error",
+        };
+        return statusMap[apiStatus.toLowerCase()] ?? "running";
+    }
+    /**
+     * Main execution loop
+     */
+    async execute() {
+        // First, resume any runs that were already in-flight
+        this.resumeInFlightRuns();
+        while (!this.stopped && !this.isComplete()) {
+            // Start new runs if we have capacity
+            while (!this.stopped && this.activeRuns.size < this.maxConcurrent && this.nextIndex < this.runs.length) {
+                const index = this.nextIndex;
+                // Skip already completed runs (from restored progress)
+                if (this.completedRuns.has(index)) {
+                    this.nextIndex++;
+                    continue;
+                }
+                this.nextIndex++;
+                // Don't await - run concurrently
+                this.startRun(index);
+                // Wait a bit to avoid overwhelming the API
+                await this.delay(150);
+            }
+            // Wait a bit before checking again
+            await this.delay(500);
+        }
+        // Wait for active runs to complete
+        while (this.activeRuns.size > 0 && !this.stopped) {
+            await this.delay(500);
+        }
+    }
+    /**
+     * Helper for delays
+     */
+    delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Get progress entries for saving
+     */
+    getProgressEntries() {
+        return this.states.map((state) => ({
+            index: state.index,
+            status: state.status,
+            runId: state.runId,
+            startTime: state.startTime,
+            endTime: this.completedRuns.has(state.index) ? Date.now() : undefined,
+            error: state.error,
+        }));
+    }
+    /**
+     * Get URL for a run
+     */
+    getRunUrl(index) {
+        const state = this.states[index];
+        if (state?.runId) {
+            return `${this.baseUrl}/runs/${state.runId}`;
+        }
+        return undefined;
+    }
+}

package/dist/lib/evaluation/tui.d.ts

@@ -0,0 +1,20 @@
+import type { RunState, StatusCounts } from "./types.js";
+export interface TUIOptions {
+    evaluationName: string;
+    onQuit: () => void;
+    onOpenRun: (index: number) => void;
+}
+export declare class TUI {
+    private options;
+    private states;
+    private statusCounts;
+    private app?;
+    private running;
+    constructor(options: TUIOptions);
+    start(): void;
+    stop(): void;
+    updateStates(states: RunState[]): void;
+    updateStatusCounts(counts: StatusCounts): void;
+    private rerender;
+    private renderApp;
+}