dsh-model-arena 0.2.1 → 0.4.0

package/lib/routes.js

@@ -1,18 +1,20 @@
-import { ARENA_PANEL_PATH, ARENA_RUN_PATH, ARENA_RATE_PATH } from './arena-view.js';
-export { ARENA_PANEL_PATH, ARENA_RUN_PATH, ARENA_RATE_PATH };
+import { ARENA_PANEL_PATH, ARENA_RUN_PATH, ARENA_RATE_PATH, ARENA_COMPARE_PATH, ARENA_DELETE_PATH } from './arena-view.js';
+export { ARENA_PANEL_PATH, ARENA_RUN_PATH, ARENA_RATE_PATH, ARENA_COMPARE_PATH, ARENA_DELETE_PATH };
 export function registerArenaRoutes(ctx, arena) {
     ctx.inject(['connection'], (connectionCtx) => {
         const connection = connectionCtx.connection;
+        // Panel payload (with leaderboard + stats)
         connection.fetch.register({
             path: ARENA_PANEL_PATH,
             methods: ['GET'],
             requestBody: 'buffered',
-            // The panel view is written against `PanelPayload`; returning the bare run list
-            // here made every array method on `payload.recentRuns` throw in the browser.
-            fetch: () => Promise.resolve(Response.json({ recentRuns: arena.listRuns() }, {
-                headers: { 'cache-control': 'no-store' },
-            })),
+            fetch: () => Promise.resolve(Response.json({
+                recentRuns: arena.listRuns(),
+                leaderboard: arena.leaderboard(),
+                stats: arena.stats(),
+            }, { headers: { 'cache-control': 'no-store' } })),
         });
+        // Run detail
         connection.fetch.register({
             path: ARENA_RUN_PATH,
             methods: ['GET'],
@@ -28,6 +30,7 @@ export function registerArenaRoutes(ctx, arena) {
                 return Response.json(run, { headers: { 'cache-control': 'no-store' } });
             },
         });
+        // Rate a model in a run
         connection.fetch.register({
             path: ARENA_RATE_PATH,
             methods: ['POST'],
@@ -49,5 +52,46 @@ export function registerArenaRoutes(ctx, arena) {
                 return Response.json(run, { headers: { 'cache-control': 'no-store' } });
             },
         });
+        // Compare two models in a run
+        connection.fetch.register({
+            path: ARENA_COMPARE_PATH,
+            methods: ['GET'],
+            requestBody: 'buffered',
+            fetch: async (request) => {
+                const url = new URL(request.url);
+                const runId = url.searchParams.get('runId');
+                const modelA = url.searchParams.get('modelA');
+                const modelB = url.searchParams.get('modelB');
+                if (!runId || !modelA || !modelB) {
+                    return Response.json({ error: 'missing runId, modelA, or modelB' }, { status: 400 });
+                }
+                const result = arena.compare(runId, modelA, modelB);
+                if (!result)
+                    return Response.json({ error: 'not found' }, { status: 404 });
+                return Response.json(result, { headers: { 'cache-control': 'no-store' } });
+            },
+        });
+        // Delete a run
+        connection.fetch.register({
+            path: ARENA_DELETE_PATH,
+            methods: ['POST'],
+            requestBody: 'buffered',
+            fetch: async (request) => {
+                let body;
+                try {
+                    body = await request.json();
+                }
+                catch {
+                    return Response.json({ error: 'request body must be JSON' }, { status: 400 });
+                }
+                if (!body.runId) {
+                    return Response.json({ error: 'missing runId' }, { status: 400 });
+                }
+                const ok = arena.deleteRun(body.runId);
+                if (!ok)
+                    return Response.json({ error: 'run not found' }, { status: 404 });
+                return Response.json({ ok: true }, { headers: { 'cache-control': 'no-store' } });
+            },
+        });
     });
 }

package/lib/scoring.d.ts

@@ -1,5 +1,10 @@
-import type { ModelResult, Score } from './types.js';
+import type { ModelResult, Score, LeaderboardEntry } from './types.js';
 /** Score outputs automatically based on simple metrics. */
 export declare function autoScore(results: ModelResult[], reference?: string): Record<string, Score>;
 /** Simple character-level similarity (Jaccard on character bigrams). */
 export declare function similarity(a: string, b: string): number;
+/** Build a leaderboard from all runs using Elo rating. */
+export declare function buildLeaderboard(runs: {
+    results: ModelResult[];
+    scores: Record<string, Score>;
+}[]): LeaderboardEntry[];

package/lib/scoring.js

@@ -33,3 +33,101 @@ export function similarity(a, b) {
             intersection++;
     return intersection / (bigramsA.size + bigramsB.size - intersection);
 }
+// ── Elo rating system ────────────────────────────────────────
+const INITIAL_ELO = 1200;
+const K_FACTOR = 32;
+/** Compute expected score (probability of winning). */
+function expectedScore(ratingA, ratingB) {
+    return 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400));
+}
+/** Build a leaderboard from all runs using Elo rating. */
+export function buildLeaderboard(runs) {
+    const records = new Map();
+    const getOrInit = (model) => {
+        if (!records.has(model)) {
+            records.set(model, { elo: INITIAL_ELO, wins: 0, losses: 0, draws: 0, games: 0, totalLatencyMs: 0, totalTokens: 0, totalRuns: 0, totalRating: 0, ratingCount: 0 });
+        }
+        return records.get(model);
+    };
+    for (const run of runs) {
+        // accumulate raw stats
+        for (const r of run.results) {
+            const rec = getOrInit(r.model);
+            rec.totalLatencyMs += r.latencyMs;
+            rec.totalTokens += r.promptTokens + r.completionTokens;
+            rec.totalRuns++;
+            if (r.rating !== undefined) {
+                rec.totalRating += r.rating;
+                rec.ratingCount++;
+            }
+        }
+        // pairwise Elo updates based on user rating (if any) or composite auto-score
+        const models = run.results.map((r) => r.model);
+        for (let i = 0; i < models.length; i++) {
+            for (let j = i + 1; j < models.length; j++) {
+                const a = models[i];
+                const b = models[j];
+                const recA = getOrInit(a);
+                const recB = getOrInit(b);
+                const sA = compositeScore(run.scores[a]);
+                const sB = compositeScore(run.scores[b]);
+                const eA = expectedScore(recA.elo, recB.elo);
+                let actualA;
+                if (Math.abs(sA - sB) < 0.01) {
+                    // draw
+                    actualA = 0.5;
+                    recA.draws++;
+                    recB.draws++;
+                }
+                else if (sA > sB) {
+                    actualA = 1;
+                    recA.wins++;
+                    recB.losses++;
+                }
+                else {
+                    actualA = 0;
+                    recA.losses++;
+                    recB.wins++;
+                }
+                recA.elo += K_FACTOR * (actualA - eA);
+                recB.elo += K_FACTOR * ((1 - actualA) - (1 - eA));
+                recA.games++;
+                recB.games++;
+            }
+        }
+    }
+    const entries = [];
+    for (const [model, rec] of records) {
+        entries.push({
+            model,
+            elo: Math.round(rec.elo),
+            wins: rec.wins,
+            losses: rec.losses,
+            draws: rec.draws,
+            games: rec.games,
+            totalRuns: rec.totalRuns,
+            avgRating: rec.ratingCount > 0 ? Math.round((rec.totalRating / rec.ratingCount) * 10) / 10 : 0,
+            avgLatencyMs: rec.games > 0 ? Math.round(rec.totalLatencyMs / rec.games) : 0,
+            avgTokens: rec.games > 0 ? Math.round(rec.totalTokens / rec.games) : 0,
+        });
+    }
+    entries.sort((a, b) => b.elo - a.elo);
+    return entries;
+}
+/** Combine score dimensions into a single number for ranking. */
+function compositeScore(s) {
+    if (!s)
+        return 0;
+    // User rating dominates if present (1-5 scale → 0-1)
+    if (s.userRating !== undefined)
+        return s.userRating / 5;
+    // Otherwise average available auto-scores
+    const parts = [];
+    if (s.referenceMatch !== undefined)
+        parts.push(s.referenceMatch);
+    if (s.lengthScore !== undefined)
+        parts.push(s.lengthScore);
+    if (s.latencyScore !== undefined)
+        parts.push(s.latencyScore);
+    return parts.length > 0 ? parts.reduce((a, b) => a + b, 0) / parts.length : 0;
+}

package/lib/store.d.ts

@@ -9,5 +9,6 @@ export declare class ArenaStore {
     listRuns(): ArenaRun[];
     saveSuite(suite: Suite): void;
     getSuite(name: string): Suite | null;
+    deleteRun(id: string): boolean;
     static generateId(prompt: string, models: string[]): string;
 }

package/lib/store.js

@@ -1,4 +1,4 @@
-import { writeFileSync, readFileSync, existsSync, mkdirSync, readdirSync } from 'node:fs';
+import { writeFileSync, readFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'node:fs';
 import { join } from 'node:path';
 import { createHash } from 'node:crypto';
 export class ArenaStore {
@@ -40,6 +40,13 @@ export class ArenaStore {
             return null;
         return JSON.parse(readFileSync(path, 'utf8'));
     }
+    deleteRun(id) {
+        const path = join(this.arenaDir, `${id}.json`);
+        if (!existsSync(path))
+            return false;
+        unlinkSync(path);
+        return true;
+    }
     static generateId(prompt, models) {
         const hash = createHash('sha256');
         hash.update(prompt);

package/lib/types.d.ts

@@ -5,6 +5,7 @@ export interface ModelResult {
     completionTokens: number;
     latencyMs: number;
     error?: string;
+    rating?: number;
 }
 export interface Score {
     userRating?: number;
@@ -18,6 +19,7 @@ export interface ArenaRun {
     timestamp: number;
     results: ModelResult[];
     scores: Record<string, Score>;
+    reference?: string | undefined;
 }
 export interface Suite {
     name: string;
@@ -36,11 +38,53 @@ export interface RegressionReport {
         currentOutput: string;
     }[];
 }
-export interface PanelPayload {
-    recentRuns: {
-        id: string;
-        prompt: string;
-        timestamp: number;
-        modelCount: number;
+export interface LeaderboardEntry {
+    model: string;
+    elo: number;
+    wins: number;
+    losses: number;
+    draws: number;
+    games: number;
+    totalRuns: number;
+    avgRating: number;
+    avgLatencyMs: number;
+    avgTokens: number;
+}
+export interface RunSummary {
+    id: string;
+    prompt: string;
+    timestamp: number;
+    modelCount: number;
+    winner?: string | undefined;
+}
+export interface ComparePayload {
+    runId: string;
+    modelA: string;
+    modelB: string;
+    outputA: string;
+    outputB: string;
+    diff: {
+        type: 'same' | 'added' | 'removed';
+        text: string;
     }[];
+    summary: {
+        added: number;
+        removed: number;
+        changed: number;
+    };
+    scoreA: Score;
+    scoreB: Score;
+}
+export interface ArenaStats {
+    totalRuns: number;
+    totalModels: number;
+    uniqueModels: number;
+    uniquePrompts: number;
+    totalRatings: number;
+    avgModelsPerRun: number;
+}
+export interface PanelPayload {
+    recentRuns: RunSummary[];
+    leaderboard: LeaderboardEntry[];
+    stats: ArenaStats;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-model-arena",
-  "version": "0.2.1",
+  "version": "0.4.0",
   "description": "Run the same prompt through every model you have, and see the difference without squinting.",
   "type": "module",
   "main": "lib/index.js",