dsh-model-arena 0.2.1 → 0.4.0

package/lib/arena-view.d.ts

@@ -1,6 +1,8 @@
-import type { PanelPayload, ArenaRun } from './types.js';
+import type { PanelPayload, RunSummary, LeaderboardEntry, ArenaStats } from './types.js';
 /** Route paths shared between host and client. */
 export declare const ARENA_PANEL_PATH = "/api/arena.panel";
 export declare const ARENA_RUN_PATH = "/api/arena.run";
 export declare const ARENA_RATE_PATH = "/api/arena.rate";
-export declare function panelState(runs: ArenaRun[]): PanelPayload;
+export declare const ARENA_COMPARE_PATH = "/api/arena.compare";
+export declare const ARENA_DELETE_PATH = "/api/arena.delete";
+export declare function panelState(runs: RunSummary[], leaderboard: LeaderboardEntry[], stats: ArenaStats): PanelPayload;

package/lib/arena-view.js

@@ -2,16 +2,8 @@
 export const ARENA_PANEL_PATH = '/api/arena.panel';
 export const ARENA_RUN_PATH = '/api/arena.run';
 export const ARENA_RATE_PATH = '/api/arena.rate';
-export function panelState(runs) {
-    return {
-        recentRuns: runs
-            .sort((a, b) => b.timestamp - a.timestamp)
-            .slice(0, 20)
-            .map((r) => ({
-            id: r.id,
-            prompt: r.prompt,
-            timestamp: r.timestamp,
-            modelCount: r.results.length,
-        })),
-    };
+export const ARENA_COMPARE_PATH = '/api/arena.compare';
+export const ARENA_DELETE_PATH = '/api/arena.delete';
+export function panelState(runs, leaderboard, stats) {
+    return { recentRuns: runs.slice(0, 20), leaderboard, stats };
 }

package/lib/arena.d.ts

@@ -1,4 +1,4 @@
-import type { ArenaRun, ModelResult, Suite, RegressionReport, Score } from './types.js';
+import type { ArenaRun, ModelResult, Suite, RegressionReport, LeaderboardEntry, ArenaStats, RunSummary, ComparePayload } from './types.js';
 export declare class Arena {
     private projectDir;
     private store;
@@ -6,31 +6,21 @@ export declare class Arena {
     /** Save a completed run (results already collected from model calls). */
     saveRun(prompt: string, results: ModelResult[], reference?: string): ArenaRun;
     getRun(id: string): ArenaRun | null;
-    listRuns(): {
-        id: string;
-        prompt: string;
-        timestamp: number;
-        modelCount: number;
-    }[];
+    listRuns(): RunSummary[];
     /** Compare two models' outputs within a run. */
-    compare(runId: string, modelA: string, modelB: string): {
-        modelA: string;
-        modelB: string;
-        outputA: string;
-        outputB: string;
-        diff: import("./diff.js").DiffLine[];
-        summary: {
-            added: number;
-            removed: number;
-            changed: number;
-        };
-        scoreA: Score;
-        scoreB: Score;
-    } | null;
+    compare(runId: string, modelA: string, modelB: string): ComparePayload | null;
     /** Set a user rating for a model in a run. */
     rate(runId: string, model: string, rating: number): ArenaRun | null;
+    /** Delete a run. */
+    deleteRun(runId: string): boolean;
     /** Save a benchmark suite. */
     saveSuite(suite: Suite): void;
     /** Generate a regression report between two runs of the same suite. */
     regression(suiteName: string, baselineRunId: string, currentRunId: string): RegressionReport | null;
+    /** Elo leaderboard from all runs. */
+    leaderboard(): LeaderboardEntry[];
+    /** Aggregate stats. */
+    stats(): ArenaStats;
+    /** Pick the best model in a run by composite score. */
+    private pickWinner;
 }

package/lib/arena.js

@@ -1,5 +1,5 @@
 import { ArenaStore } from './store.js';
-import { autoScore } from './scoring.js';
+import { autoScore, buildLeaderboard } from './scoring.js';
 import { lineDiff, diffSummary } from './diff.js';
 export class Arena {
     projectDir;
@@ -13,7 +13,7 @@ export class Arena {
         const models = results.map((r) => r.model);
         const id = ArenaStore.generateId(prompt, models);
         const scores = autoScore(results, reference);
-        const run = { id, prompt, timestamp: Date.now(), results, scores };
+        const run = { id, prompt, timestamp: Date.now(), results, scores, reference };
         this.store.saveRun(run);
         return run;
     }
@@ -21,12 +21,16 @@ export class Arena {
         return this.store.getRun(id);
     }
     listRuns() {
-        return this.store.listRuns().map((r) => ({
-            id: r.id,
-            prompt: r.prompt,
-            timestamp: r.timestamp,
-            modelCount: r.results.length,
-        }));
+        return this.store.listRuns().map((r) => {
+            const winner = this.pickWinner(r);
+            return {
+                id: r.id,
+                prompt: r.prompt,
+                timestamp: r.timestamp,
+                modelCount: r.results.length,
+                winner,
+            };
+        });
     }
     /** Compare two models' outputs within a run. */
     compare(runId, modelA, modelB) {
@@ -38,14 +42,15 @@ export class Arena {
         if (!a || !b)
             return null;
         return {
+            runId,
             modelA: a.model,
             modelB: b.model,
             outputA: a.output,
             outputB: b.output,
             diff: lineDiff(a.output, b.output),
             summary: diffSummary(a.output, b.output),
-            scoreA: run.scores[modelA],
-            scoreB: run.scores[modelB],
+            scoreA: run.scores[modelA] ?? {},
+            scoreB: run.scores[modelB] ?? {},
         };
     }
     /** Set a user rating for a model in a run. */
@@ -59,6 +64,10 @@ export class Arena {
         this.store.saveRun(run);
         return run;
     }
+    /** Delete a run. */
+    deleteRun(runId) {
+        return this.store.deleteRun(runId);
+    }
     /** Save a benchmark suite. */
     saveSuite(suite) {
         this.store.saveSuite(suite);
@@ -84,4 +93,51 @@ export class Arena {
         }
         return { suiteName, baselineRunId, currentRunId, changes };
     }
+    /** Elo leaderboard from all runs. */
+    leaderboard() {
+        const allRuns = this.store.listRuns();
+        return buildLeaderboard(allRuns);
+    }
+    /** Aggregate stats. */
+    stats() {
+        const allRuns = this.store.listRuns();
+        const allModels = new Set();
+        const allPrompts = new Set();
+        for (const run of allRuns) {
+            allPrompts.add(run.prompt);
+            for (const r of run.results)
+                allModels.add(r.model);
+        }
+        let totalRatings = 0;
+        for (const run of allRuns) {
+            for (const [, score] of Object.entries(run.scores)) {
+                if (score.userRating !== undefined)
+                    totalRatings++;
+            }
+        }
+        return {
+            totalRuns: allRuns.length,
+            totalModels: allModels.size,
+            uniqueModels: allModels.size,
+            uniquePrompts: allPrompts.size,
+            totalRatings,
+            avgModelsPerRun: allRuns.length > 0
+                ? Math.round((allRuns.reduce((s, r) => s + r.results.length, 0) / allRuns.length) * 10) / 10
+                : 0,
+        };
+    }
+    /** Pick the best model in a run by composite score. */
+    pickWinner(run) {
+        let best;
+        let bestScore = -Infinity;
+        for (const [model, score] of Object.entries(run.scores)) {
+            const s = (score.userRating !== undefined ? score.userRating / 5 : 0)
+                + (score.referenceMatch ?? 0) + (score.latencyScore ?? 0);
+            if (s > bestScore) {
+                bestScore = s;
+                best = model;
+            }
+        }
+        return best;
+    }
 }