@hemansubedi/aether-ai 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitattributes +3 -0
- package/.github/workflows/live-stats.yml +42 -0
- package/.github/workflows/publish.yml +34 -0
- package/.github/workflows/update-preview.yml +41 -0
- package/INSTALL.md +59 -0
- package/LICENSE +21 -0
- package/README.md +397 -0
- package/assets/aether-arena.svg +72 -0
- package/assets/aether-banner.svg +62 -0
- package/assets/aether-router.svg +129 -0
- package/dist/agent.js +125 -0
- package/dist/arena.js +486 -0
- package/dist/checkpoint.js +105 -0
- package/dist/client.js +95 -0
- package/dist/combos.js +176 -0
- package/dist/commands.js +483 -0
- package/dist/config.js +104 -0
- package/dist/cost.js +176 -0
- package/dist/git.js +52 -0
- package/dist/health.js +81 -0
- package/dist/index.js +272 -0
- package/dist/keys.js +128 -0
- package/dist/memory.js +98 -0
- package/dist/modes.js +68 -0
- package/dist/providers/index.js +32 -0
- package/dist/providers/ollama.js +206 -0
- package/dist/providers/openai-compat.js +181 -0
- package/dist/providers/openrouter.js +189 -0
- package/dist/providers/registry.js +211 -0
- package/dist/router-engine.js +200 -0
- package/dist/router.js +171 -0
- package/dist/server.js +210 -0
- package/dist/session.js +97 -0
- package/dist/settings.js +97 -0
- package/dist/skills.js +100 -0
- package/dist/tokensaver.js +50 -0
- package/dist/tools/filesystem.js +243 -0
- package/dist/tools/git.js +53 -0
- package/dist/tools/glob.js +175 -0
- package/dist/tools/grep.js +193 -0
- package/dist/tools/registry.js +39 -0
- package/dist/tools/vision.js +140 -0
- package/dist/tools/websearch.js +118 -0
- package/dist/tui.js +562 -0
- package/dist/types.js +8 -0
- package/docs/preview.txt +51 -0
- package/docs/screenshots.md +110 -0
- package/docs/stats.md +5 -0
- package/install.ps1 +170 -0
- package/install.sh +196 -0
- package/package.json +34 -0
- package/scripts/generate-stats-card.ts +62 -0
- package/scripts/patch_index.ps1 +17 -0
- package/scripts/release.sh +7 -0
- package/src/agent.ts +146 -0
- package/src/arena.ts +584 -0
- package/src/checkpoint.ts +111 -0
- package/src/client.ts +172 -0
- package/src/combos.ts +199 -0
- package/src/commands.ts +973 -0
- package/src/config.ts +122 -0
- package/src/cost.ts +206 -0
- package/src/git.ts +68 -0
- package/src/health.ts +90 -0
- package/src/index.ts +281 -0
- package/src/keys.ts +135 -0
- package/src/memory.ts +101 -0
- package/src/modes.ts +84 -0
- package/src/providers/index.ts +59 -0
- package/src/providers/ollama.ts +222 -0
- package/src/providers/openai-compat.ts +188 -0
- package/src/providers/openrouter.ts +198 -0
- package/src/providers/registry.ts +223 -0
- package/src/router-engine.ts +214 -0
- package/src/router.ts +195 -0
- package/src/server.ts +242 -0
- package/src/session.ts +111 -0
- package/src/settings.ts +125 -0
- package/src/skills.ts +106 -0
- package/src/tokensaver.ts +57 -0
- package/src/tools/filesystem.ts +258 -0
- package/src/tools/git.ts +53 -0
- package/src/tools/glob.ts +180 -0
- package/src/tools/grep.ts +192 -0
- package/src/tools/registry.ts +54 -0
- package/src/tools/vision.ts +152 -0
- package/src/tools/websearch.ts +130 -0
- package/src/tui.ts +664 -0
- package/src/types.ts +77 -0
- package/tsconfig.json +16 -0
package/dist/arena.js
ADDED
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
const ARENA_FILE = path.join(os.homedir(), ".aether", "arena.json");
|
|
5
|
+
const ARENA_DIR = path.dirname(ARENA_FILE);
|
|
6
|
+
const LEADERBOARD_JSON = path.join(ARENA_DIR, "leaderboard.json");
|
|
7
|
+
const LEADERBOARD_MD = path.join(ARENA_DIR, "leaderboard.md");
|
|
8
|
+
const K_FACTOR = 32;
|
|
9
|
+
export class Arena {
|
|
10
|
+
router;
|
|
11
|
+
elo = new Map();
|
|
12
|
+
wins = new Map();
|
|
13
|
+
losses = new Map();
|
|
14
|
+
headToHead = new Map();
|
|
15
|
+
constructor(router) {
|
|
16
|
+
this.router = router;
|
|
17
|
+
this.load();
|
|
18
|
+
}
|
|
19
|
+
ensure(modelId) {
|
|
20
|
+
if (!this.elo.has(modelId))
|
|
21
|
+
this.elo.set(modelId, 1200);
|
|
22
|
+
if (!this.wins.has(modelId))
|
|
23
|
+
this.wins.set(modelId, 0);
|
|
24
|
+
if (!this.losses.has(modelId))
|
|
25
|
+
this.losses.set(modelId, 0);
|
|
26
|
+
}
|
|
27
|
+
/** Run the same prompt through N models in parallel, streaming each result. */
|
|
28
|
+
async compare(userMessage, history, modelIds, onResult) {
|
|
29
|
+
const settled = await Promise.all(modelIds.map((modelId) => this.runOne(modelId, userMessage, history, onResult)));
|
|
30
|
+
return settled.filter((r) => r !== null);
|
|
31
|
+
}
|
|
32
|
+
async runOne(modelId, userMessage, history, onResult) {
|
|
33
|
+
const start = Date.now();
|
|
34
|
+
let text = "";
|
|
35
|
+
const toolCalls = [];
|
|
36
|
+
let usage;
|
|
37
|
+
let error;
|
|
38
|
+
try {
|
|
39
|
+
const provider = await this.router.getModelProvider(modelId);
|
|
40
|
+
const messages = [...history, { role: "user", content: userMessage }];
|
|
41
|
+
for await (const chunk of provider.chat(messages, [], { temperature: 0.7, maxTokens: 4096 })) {
|
|
42
|
+
onResult(modelId, chunk);
|
|
43
|
+
if (chunk.type === "text" && chunk.text)
|
|
44
|
+
text += chunk.text;
|
|
45
|
+
if (chunk.type === "tool_call" && chunk.tool_call)
|
|
46
|
+
toolCalls.push(chunk.tool_call);
|
|
47
|
+
if (chunk.type === "done")
|
|
48
|
+
usage = chunk.usage;
|
|
49
|
+
if (chunk.type === "error" && chunk.error)
|
|
50
|
+
error = chunk.error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
error = err.message;
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
modelId,
|
|
58
|
+
text,
|
|
59
|
+
toolCalls,
|
|
60
|
+
usage,
|
|
61
|
+
error,
|
|
62
|
+
elapsedMs: Date.now() - start,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Collect full responses from each model for blind voting. */
|
|
66
|
+
async vote(userMessage, history, modelIds) {
|
|
67
|
+
const results = await this.compare(userMessage, history, modelIds, () => { });
|
|
68
|
+
return results.map((r) => ({ modelId: r.modelId, response: r.error ? `[error] ${r.error}` : r.text }));
|
|
69
|
+
}
|
|
70
|
+
recordMatch(winner, loser) {
|
|
71
|
+
this.ensure(winner);
|
|
72
|
+
this.ensure(loser);
|
|
73
|
+
const eW = this.elo.get(winner);
|
|
74
|
+
const eL = this.elo.get(loser);
|
|
75
|
+
const expectedW = 1 / (1 + Math.pow(10, (eL - eW) / 400));
|
|
76
|
+
const expectedL = 1 - expectedW;
|
|
77
|
+
this.elo.set(winner, Math.round(eW + K_FACTOR * (1 - expectedW)));
|
|
78
|
+
this.elo.set(loser, Math.round(eL + K_FACTOR * (0 - expectedL)));
|
|
79
|
+
this.wins.set(winner, (this.wins.get(winner) ?? 0) + 1);
|
|
80
|
+
this.losses.set(loser, (this.losses.get(loser) ?? 0) + 1);
|
|
81
|
+
this.save();
|
|
82
|
+
}
|
|
83
|
+
getRankings() {
|
|
84
|
+
const out = [];
|
|
85
|
+
for (const modelId of this.elo.keys()) {
|
|
86
|
+
this.ensure(modelId);
|
|
87
|
+
out.push({
|
|
88
|
+
modelId,
|
|
89
|
+
elo: this.elo.get(modelId),
|
|
90
|
+
wins: this.wins.get(modelId),
|
|
91
|
+
losses: this.losses.get(modelId),
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
return out.sort((a, b) => b.elo - a.elo);
|
|
95
|
+
}
|
|
96
|
+
/** Render a side-by-side comparison string for display. */
|
|
97
|
+
render(results) {
|
|
98
|
+
if (results.length === 0)
|
|
99
|
+
return "(no results)";
|
|
100
|
+
const cols = results.map((r) => ({
|
|
101
|
+
modelId: r.modelId,
|
|
102
|
+
text: r.error ? `⚠ ${r.text}` : r.text,
|
|
103
|
+
error: r.error,
|
|
104
|
+
elapsedMs: r.elapsedMs,
|
|
105
|
+
}));
|
|
106
|
+
const width = Math.max(20, ...cols.map((c) => c.modelId.length + 4));
|
|
107
|
+
const truncate = (s, max) => {
|
|
108
|
+
if (s.length <= max)
|
|
109
|
+
return s;
|
|
110
|
+
return s.slice(0, max - 1) + "\u2026";
|
|
111
|
+
};
|
|
112
|
+
const header = cols
|
|
113
|
+
.map((c) => truncate(c.modelId, width))
|
|
114
|
+
.join(" \u2502 ");
|
|
115
|
+
const sep = cols.map(() => "-".repeat(width)).join("-+-");
|
|
116
|
+
const maxLen = Math.max(...cols.map((c) => c.text.split("\n").length));
|
|
117
|
+
const body = [];
|
|
118
|
+
for (let i = 0; i < maxLen; i++) {
|
|
119
|
+
const row = cols
|
|
120
|
+
.map((c) => {
|
|
121
|
+
const src = c.text.split("\n")[i] ?? "";
|
|
122
|
+
return truncate(src, width).padEnd(width);
|
|
123
|
+
})
|
|
124
|
+
.join(" \u2502 ");
|
|
125
|
+
body.push(row);
|
|
126
|
+
}
|
|
127
|
+
const footer = cols
|
|
128
|
+
.map((c) => `${truncate(c.modelId, width)}: ${(c.elapsedMs / 1000).toFixed(2)}s`)
|
|
129
|
+
.join(" \u2502 ");
|
|
130
|
+
const lines = [header, sep, ...body, "", footer];
|
|
131
|
+
return lines.join("\n");
|
|
132
|
+
}
|
|
133
|
+
splitColumns(results, width) {
|
|
134
|
+
const maxLen = Math.max(...results.map((r) => r.text.split("\n").length));
|
|
135
|
+
const lines = [];
|
|
136
|
+
for (let i = 0; i < maxLen; i++) {
|
|
137
|
+
const cols = results.map((r) => {
|
|
138
|
+
const src = r.text.split("\n")[i] ?? "";
|
|
139
|
+
return (r.error ? `> ${src}` : src).padEnd(width);
|
|
140
|
+
});
|
|
141
|
+
lines.push(cols.join(" | "));
|
|
142
|
+
}
|
|
143
|
+
return lines;
|
|
144
|
+
}
|
|
145
|
+
load() {
|
|
146
|
+
try {
|
|
147
|
+
if (!fs.existsSync(ARENA_FILE))
|
|
148
|
+
return;
|
|
149
|
+
const raw = fs.readFileSync(ARENA_FILE, "utf8");
|
|
150
|
+
const obj = JSON.parse(raw);
|
|
151
|
+
if (obj && typeof obj === "object") {
|
|
152
|
+
if (obj.elo && typeof obj.elo === "object") {
|
|
153
|
+
for (const [k, v] of Object.entries(obj.elo))
|
|
154
|
+
this.elo.set(k, Number(v) || 1200);
|
|
155
|
+
}
|
|
156
|
+
if (obj.wins && typeof obj.wins === "object") {
|
|
157
|
+
for (const [k, v] of Object.entries(obj.wins))
|
|
158
|
+
this.wins.set(k, Number(v) || 0);
|
|
159
|
+
}
|
|
160
|
+
if (obj.losses && typeof obj.losses === "object") {
|
|
161
|
+
for (const [k, v] of Object.entries(obj.losses))
|
|
162
|
+
this.losses.set(k, Number(v) || 0);
|
|
163
|
+
}
|
|
164
|
+
if (obj.headToHead && typeof obj.headToHead === "object") {
|
|
165
|
+
for (const [k, v] of Object.entries(obj.headToHead)) {
|
|
166
|
+
const h = v;
|
|
167
|
+
this.headToHead.set(k, {
|
|
168
|
+
winsA: Number(h.winsA) || 0,
|
|
169
|
+
winsB: Number(h.winsB) || 0,
|
|
170
|
+
ties: Number(h.ties) || 0,
|
|
171
|
+
total: Number(h.total) || 0,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
// ignore malformed arena file
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
save() {
|
|
182
|
+
try {
|
|
183
|
+
if (!fs.existsSync(ARENA_DIR))
|
|
184
|
+
fs.mkdirSync(ARENA_DIR, { recursive: true });
|
|
185
|
+
const tmp = ARENA_FILE + ".tmp";
|
|
186
|
+
fs.writeFileSync(tmp, JSON.stringify({
|
|
187
|
+
elo: Object.fromEntries(this.elo),
|
|
188
|
+
wins: Object.fromEntries(this.wins),
|
|
189
|
+
losses: Object.fromEntries(this.losses),
|
|
190
|
+
headToHead: Object.fromEntries(this.headToHead),
|
|
191
|
+
}, null, 2), "utf8");
|
|
192
|
+
fs.renameSync(tmp, ARENA_FILE);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
// best-effort persistence
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// ---------------------------------------------------------------------------
|
|
199
|
+
// Head-to-head
|
|
200
|
+
// ---------------------------------------------------------------------------
|
|
201
|
+
/** Track direct confrontations between two specific models. */
|
|
202
|
+
h2h(modelA, modelB) {
|
|
203
|
+
const key = this.h2hKey(modelA, modelB);
|
|
204
|
+
const raw = this.headToHead.get(key);
|
|
205
|
+
if (raw)
|
|
206
|
+
return { ...raw };
|
|
207
|
+
return { winsA: 0, winsB: 0, ties: 0, total: 0 };
|
|
208
|
+
}
|
|
209
|
+
h2hKey(modelA, modelB) {
|
|
210
|
+
return `${modelA}::${modelB}`;
|
|
211
|
+
}
|
|
212
|
+
recordH2H(modelA, modelB, winner) {
|
|
213
|
+
const key = this.h2hKey(modelA, modelB);
|
|
214
|
+
const cur = this.headToHead.get(key) ?? { winsA: 0, winsB: 0, ties: 0, total: 0 };
|
|
215
|
+
cur.total += 1;
|
|
216
|
+
if (winner === "A")
|
|
217
|
+
cur.winsA += 1;
|
|
218
|
+
else if (winner === "B")
|
|
219
|
+
cur.winsB += 1;
|
|
220
|
+
else
|
|
221
|
+
cur.ties += 1;
|
|
222
|
+
this.headToHead.set(key, cur);
|
|
223
|
+
}
|
|
224
|
+
// ---------------------------------------------------------------------------
|
|
225
|
+
// Leaderboard export
|
|
226
|
+
// ---------------------------------------------------------------------------
|
|
227
|
+
/** Export all elo, wins, losses, and rankings as a JSON string. */
|
|
228
|
+
exportJSON() {
|
|
229
|
+
const rankings = this.getRankings();
|
|
230
|
+
const payload = {
|
|
231
|
+
generatedAt: Date.now(),
|
|
232
|
+
rankings: rankings.map((r) => ({
|
|
233
|
+
modelId: r.modelId,
|
|
234
|
+
elo: r.elo,
|
|
235
|
+
wins: r.wins,
|
|
236
|
+
losses: r.losses,
|
|
237
|
+
winRate: r.wins + r.losses > 0 ? r.wins / (r.wins + r.losses) : 0,
|
|
238
|
+
})),
|
|
239
|
+
elo: Object.fromEntries(this.elo),
|
|
240
|
+
wins: Object.fromEntries(this.wins),
|
|
241
|
+
losses: Object.fromEntries(this.losses),
|
|
242
|
+
headToHead: Object.fromEntries(this.headToHead),
|
|
243
|
+
};
|
|
244
|
+
return JSON.stringify(payload, null, 2);
|
|
245
|
+
}
|
|
246
|
+
/** Export the leaderboard as a markdown table. */
|
|
247
|
+
exportMarkdown() {
|
|
248
|
+
const rankings = this.getRankings();
|
|
249
|
+
if (rankings.length === 0) {
|
|
250
|
+
return "| Rank | Model | Elo | Wins | Losses | Winrate |\n|------|-------|-----|------|--------|--------|\n";
|
|
251
|
+
}
|
|
252
|
+
const lines = [];
|
|
253
|
+
lines.push("| Rank | Model | Elo | Wins | Losses | Winrate |");
|
|
254
|
+
lines.push("|------|-------|-----|------|--------|--------|");
|
|
255
|
+
rankings.forEach((r, i) => {
|
|
256
|
+
const total = r.wins + r.losses;
|
|
257
|
+
const winrate = total > 0 ? (r.wins / total) * 100 : 0;
|
|
258
|
+
lines.push(`| ${i + 1} | ${r.modelId} | ${r.elo} | ${r.wins} | ${r.losses} | ${winrate.toFixed(1)}% |`);
|
|
259
|
+
});
|
|
260
|
+
return lines.join("\n");
|
|
261
|
+
}
|
|
262
|
+
/** Write the leaderboard to disk in the requested format. */
|
|
263
|
+
saveLeaderboard(format) {
|
|
264
|
+
try {
|
|
265
|
+
if (!fs.existsSync(ARENA_DIR))
|
|
266
|
+
fs.mkdirSync(ARENA_DIR, { recursive: true });
|
|
267
|
+
const target = format === "json" ? LEADERBOARD_JSON : LEADERBOARD_MD;
|
|
268
|
+
const content = format === "json" ? this.exportJSON() : this.exportMarkdown();
|
|
269
|
+
fs.writeFileSync(target, content, "utf8");
|
|
270
|
+
}
|
|
271
|
+
catch {
|
|
272
|
+
// best-effort persistence
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
/** Read the leaderboard back from disk. Returns null if missing/invalid. */
|
|
276
|
+
static loadLeaderboard() {
|
|
277
|
+
try {
|
|
278
|
+
if (!fs.existsSync(LEADERBOARD_JSON))
|
|
279
|
+
return null;
|
|
280
|
+
const raw = fs.readFileSync(LEADERBOARD_JSON, "utf8");
|
|
281
|
+
const obj = JSON.parse(raw);
|
|
282
|
+
if (!obj || typeof obj !== "object")
|
|
283
|
+
return null;
|
|
284
|
+
const rankings = Array.isArray(obj.rankings)
|
|
285
|
+
? obj.rankings.map((r) => ({
|
|
286
|
+
modelId: String(r.modelId),
|
|
287
|
+
elo: Number(r.elo) || 0,
|
|
288
|
+
wins: Number(r.wins) || 0,
|
|
289
|
+
losses: Number(r.losses) || 0,
|
|
290
|
+
}))
|
|
291
|
+
: [];
|
|
292
|
+
return {
|
|
293
|
+
rankings,
|
|
294
|
+
elo: (obj.elo && typeof obj.elo === "object") ? obj.elo : {},
|
|
295
|
+
wins: (obj.wins && typeof obj.wins === "object") ? obj.wins : {},
|
|
296
|
+
losses: (obj.losses && typeof obj.losses === "object") ? obj.losses : {},
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
return null;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
// ---------------------------------------------------------------------------
|
|
304
|
+
// Judge
|
|
305
|
+
// ---------------------------------------------------------------------------
|
|
306
|
+
/**
|
|
307
|
+
* Ask a judge model to score two responses on a 0-10 scale across criteria.
|
|
308
|
+
* Returns structured JSON; falls back to a tie if the judge fails.
|
|
309
|
+
*/
|
|
310
|
+
async judgeMatch(prompt, responseA, responseB, modelA, modelB, judgeModel) {
|
|
311
|
+
const judgePrompt = [
|
|
312
|
+
"You are an impartial judge for an AI model arena.",
|
|
313
|
+
"Score the two responses below on a 0-10 scale for each criterion.",
|
|
314
|
+
"Criteria: correctness, clarity, completeness, helpfulness.",
|
|
315
|
+
"Return ONLY valid JSON, no markdown fences, no prose.",
|
|
316
|
+
"Schema:",
|
|
317
|
+
"{",
|
|
318
|
+
' "scores": {',
|
|
319
|
+
' "modelA": { "overall": number, "correctness": number, "clarity": number, "completeness": number, "helpfulness": number },',
|
|
320
|
+
' "modelB": { "overall": number, "correctness": number, "clarity": number, "completeness": number, "helpfulness": number }',
|
|
321
|
+
" },",
|
|
322
|
+
' "winner": "modelA" | "modelB" | "tie",',
|
|
323
|
+
' "reasoning": "string"',
|
|
324
|
+
"}",
|
|
325
|
+
"",
|
|
326
|
+
`Prompt: ${prompt}`,
|
|
327
|
+
"",
|
|
328
|
+
`Model A (${modelA}):`,
|
|
329
|
+
responseA,
|
|
330
|
+
"",
|
|
331
|
+
`Model B (${modelB}):`,
|
|
332
|
+
responseB,
|
|
333
|
+
].join("\n");
|
|
334
|
+
const fallback = {
|
|
335
|
+
scores: {
|
|
336
|
+
modelA: { overall: 5, correctness: 5, clarity: 5, completeness: 5, helpfulness: 5 },
|
|
337
|
+
modelB: { overall: 5, correctness: 5, clarity: 5, completeness: 5, helpfulness: 5 },
|
|
338
|
+
},
|
|
339
|
+
winner: "tie",
|
|
340
|
+
reasoning: "Judge unavailable; declared a tie.",
|
|
341
|
+
};
|
|
342
|
+
try {
|
|
343
|
+
const provider = await this.router.getModelProvider(judgeModel);
|
|
344
|
+
const messages = [{ role: "user", content: judgePrompt }];
|
|
345
|
+
let text = "";
|
|
346
|
+
for await (const chunk of provider.chat(messages, [], { temperature: 0, maxTokens: 1024 })) {
|
|
347
|
+
if (chunk.type === "text" && chunk.text)
|
|
348
|
+
text += chunk.text;
|
|
349
|
+
if (chunk.type === "error")
|
|
350
|
+
throw new Error(chunk.error ?? "judge error");
|
|
351
|
+
}
|
|
352
|
+
return this.parseJudgeResponse(text, modelA, modelB);
|
|
353
|
+
}
|
|
354
|
+
catch {
|
|
355
|
+
return fallback;
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
parseJudgeResponse(raw, modelA, modelB) {
|
|
359
|
+
let cleaned = (raw ?? "").trim();
|
|
360
|
+
cleaned = cleaned.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "").trim();
|
|
361
|
+
const start = cleaned.indexOf("{");
|
|
362
|
+
const end = cleaned.lastIndexOf("}");
|
|
363
|
+
if (start === -1 || end === -1 || end <= start) {
|
|
364
|
+
throw new Error("Judge returned no JSON object");
|
|
365
|
+
}
|
|
366
|
+
const jsonText = cleaned.slice(start, end + 1);
|
|
367
|
+
const parsed = JSON.parse(jsonText);
|
|
368
|
+
if (!parsed || typeof parsed !== "object" || !parsed.scores) {
|
|
369
|
+
throw new Error("Judge JSON missing scores");
|
|
370
|
+
}
|
|
371
|
+
const normalize = (s) => ({
|
|
372
|
+
overall: this.clampScore(s?.overall),
|
|
373
|
+
correctness: this.clampScore(s?.correctness),
|
|
374
|
+
clarity: this.clampScore(s?.clarity),
|
|
375
|
+
completeness: this.clampScore(s?.completeness),
|
|
376
|
+
helpfulness: this.clampScore(s?.helpfulness),
|
|
377
|
+
});
|
|
378
|
+
const winnerRaw = String(parsed.winner ?? "tie").toLowerCase();
|
|
379
|
+
let winner;
|
|
380
|
+
if (winnerRaw === "modela" || winnerRaw === "model_a" || winnerRaw === "a")
|
|
381
|
+
winner = "modelA";
|
|
382
|
+
else if (winnerRaw === "modelb" || winnerRaw === "model_b" || winnerRaw === "b")
|
|
383
|
+
winner = "modelB";
|
|
384
|
+
else
|
|
385
|
+
winner = "tie";
|
|
386
|
+
return {
|
|
387
|
+
scores: {
|
|
388
|
+
modelA: normalize(parsed.scores.modelA),
|
|
389
|
+
modelB: normalize(parsed.scores.modelB),
|
|
390
|
+
},
|
|
391
|
+
winner,
|
|
392
|
+
reasoning: String(parsed.reasoning ?? ""),
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
clampScore(v) {
|
|
396
|
+
const n = Number(v);
|
|
397
|
+
if (!Number.isFinite(n))
|
|
398
|
+
return 5;
|
|
399
|
+
return Math.max(0, Math.min(10, n));
|
|
400
|
+
}
|
|
401
|
+
// ---------------------------------------------------------------------------
|
|
402
|
+
// Tournament
|
|
403
|
+
// ---------------------------------------------------------------------------
|
|
404
|
+
/**
|
|
405
|
+
* Run a round-robin tournament: every model plays against every other model
|
|
406
|
+
* on the same prompt. A judge model scores each pair of responses and the
|
|
407
|
+
* winner is recorded via recordMatch. Returns ArenaResult[] plus a summary.
|
|
408
|
+
*/
|
|
409
|
+
async tournament(userMessage, history, modelIds, onResult) {
|
|
410
|
+
const unique = Array.from(new Set(modelIds.filter((m) => m && m.trim())));
|
|
411
|
+
if (unique.length < 2) {
|
|
412
|
+
throw new Error("Tournament requires at least 2 distinct models");
|
|
413
|
+
}
|
|
414
|
+
const judgeModel = this.pickJudge(unique);
|
|
415
|
+
// 1. Collect full responses for every model once.
|
|
416
|
+
const responses = await this.compare(userMessage, history, unique, onResult ?? (() => { }));
|
|
417
|
+
const byModel = new Map();
|
|
418
|
+
for (const r of responses)
|
|
419
|
+
byModel.set(r.modelId, r);
|
|
420
|
+
// 2. Round-robin: every pair plays once.
|
|
421
|
+
const results = [];
|
|
422
|
+
const pairScores = new Map(); // modelId -> accumulated judge score
|
|
423
|
+
for (const m of unique)
|
|
424
|
+
pairScores.set(m, 0);
|
|
425
|
+
for (let i = 0; i < unique.length; i++) {
|
|
426
|
+
for (let j = i + 1; j < unique.length; j++) {
|
|
427
|
+
const a = unique[i];
|
|
428
|
+
const b = unique[j];
|
|
429
|
+
const ra = byModel.get(a)?.text ?? "";
|
|
430
|
+
const rb = byModel.get(b)?.text ?? "";
|
|
431
|
+
const verdict = await this.judgeMatch(userMessage, ra, rb, a, b, judgeModel);
|
|
432
|
+
const scoreA = this.average(verdict.scores.modelA);
|
|
433
|
+
const scoreB = this.average(verdict.scores.modelB);
|
|
434
|
+
pairScores.set(a, (pairScores.get(a) ?? 0) + scoreA);
|
|
435
|
+
pairScores.set(b, (pairScores.get(b) ?? 0) + scoreB);
|
|
436
|
+
if (verdict.winner === "modelA") {
|
|
437
|
+
this.recordMatch(a, b);
|
|
438
|
+
this.recordH2H(a, b, "A");
|
|
439
|
+
}
|
|
440
|
+
else if (verdict.winner === "modelB") {
|
|
441
|
+
this.recordMatch(b, a);
|
|
442
|
+
this.recordH2H(a, b, "B");
|
|
443
|
+
}
|
|
444
|
+
else {
|
|
445
|
+
this.recordH2H(a, b, "tie");
|
|
446
|
+
}
|
|
447
|
+
results.push(...responses.filter((r) => r.modelId === a || r.modelId === b));
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
const rankings = this.getRankings().map((r) => ({
|
|
451
|
+
...r,
|
|
452
|
+
elo: r.elo,
|
|
453
|
+
}));
|
|
454
|
+
// Sort by accumulated judge score, falling back to elo.
|
|
455
|
+
const sorted = [...rankings].sort((x, y) => {
|
|
456
|
+
const d = (pairScores.get(y.modelId) ?? 0) - (pairScores.get(x.modelId) ?? 0);
|
|
457
|
+
if (d !== 0)
|
|
458
|
+
return d;
|
|
459
|
+
return y.elo - x.elo;
|
|
460
|
+
});
|
|
461
|
+
const summary = {
|
|
462
|
+
prompt: userMessage,
|
|
463
|
+
matches: unique.length * (unique.length - 1) / 2,
|
|
464
|
+
rankings: sorted,
|
|
465
|
+
judge: judgeModel,
|
|
466
|
+
results,
|
|
467
|
+
};
|
|
468
|
+
return { results, summary };
|
|
469
|
+
}
|
|
470
|
+
average(score) {
|
|
471
|
+
return (score.overall + score.correctness + score.clarity + score.completeness + score.helpfulness) / 5;
|
|
472
|
+
}
|
|
473
|
+
pickJudge(modelIds) {
|
|
474
|
+
// Strongest available = highest current elo, otherwise first model.
|
|
475
|
+
let best = modelIds[0];
|
|
476
|
+
let bestElo = -Infinity;
|
|
477
|
+
for (const m of modelIds) {
|
|
478
|
+
const e = this.elo.get(m) ?? 1200;
|
|
479
|
+
if (e > bestElo) {
|
|
480
|
+
bestElo = e;
|
|
481
|
+
best = m;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return best;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import * as os from "node:os";
|
|
4
|
+
export class Checkpoint {
|
|
5
|
+
dir;
|
|
6
|
+
index;
|
|
7
|
+
constructor(dir) {
|
|
8
|
+
this.dir = dir ?? path.join(os.homedir(), ".aether", "checkpoints");
|
|
9
|
+
this.index = new Map();
|
|
10
|
+
this.loadAll();
|
|
11
|
+
}
|
|
12
|
+
get filePath() {
|
|
13
|
+
return { path: path.join(this.dir, "index.json"), entries: [] };
|
|
14
|
+
}
|
|
15
|
+
loadAll() {
|
|
16
|
+
try {
|
|
17
|
+
if (!fs.existsSync(this.dir))
|
|
18
|
+
return;
|
|
19
|
+
const indexFile = path.join(this.dir, "index.json");
|
|
20
|
+
if (!fs.existsSync(indexFile))
|
|
21
|
+
return;
|
|
22
|
+
const raw = fs.readFileSync(indexFile, "utf8");
|
|
23
|
+
const arr = JSON.parse(raw);
|
|
24
|
+
if (Array.isArray(arr)) {
|
|
25
|
+
for (const e of arr)
|
|
26
|
+
this.index.set(e.label, e);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
// ignore
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
saveIndex() {
|
|
34
|
+
try {
|
|
35
|
+
fs.mkdirSync(this.dir, { recursive: true });
|
|
36
|
+
const arr = Array.from(this.index.values());
|
|
37
|
+
const tmp = path.join(this.dir, "index.json.tmp");
|
|
38
|
+
fs.writeFileSync(tmp, JSON.stringify(arr, null, 2), "utf8");
|
|
39
|
+
fs.renameSync(tmp, path.join(this.dir, "index.json"));
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
// best effort
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
save(filePaths, label) {
|
|
46
|
+
const files = [];
|
|
47
|
+
for (const fp of filePaths) {
|
|
48
|
+
try {
|
|
49
|
+
if (fs.existsSync(fp)) {
|
|
50
|
+
files.push({ path: fp, content: fs.readFileSync(fp, "utf8") });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
// skip unreadable files
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const entry = { label, timestamp: Date.now(), files };
|
|
58
|
+
this.index.set(label, entry);
|
|
59
|
+
this.saveIndex();
|
|
60
|
+
return entry;
|
|
61
|
+
}
|
|
62
|
+
restore(label) {
|
|
63
|
+
const entry = this.index.get(label);
|
|
64
|
+
if (!entry)
|
|
65
|
+
return `ERROR: no checkpoint named "${label}"`;
|
|
66
|
+
let restored = 0;
|
|
67
|
+
for (const f of entry.files) {
|
|
68
|
+
try {
|
|
69
|
+
const dir = path.dirname(f.path);
|
|
70
|
+
if (!fs.existsSync(dir))
|
|
71
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
72
|
+
fs.writeFileSync(f.path, f.content, "utf8");
|
|
73
|
+
restored++;
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
return `ERROR restoring ${f.path}: ${err.message}`;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return `Restored ${restored} file(s) from checkpoint "${label}".`;
|
|
80
|
+
}
|
|
81
|
+
list() {
|
|
82
|
+
return Array.from(this.index.values())
|
|
83
|
+
.map((e) => ({ label: e.label, timestamp: e.timestamp, fileCount: e.files.length }))
|
|
84
|
+
.sort((a, b) => b.timestamp - a.timestamp);
|
|
85
|
+
}
|
|
86
|
+
latest() {
|
|
87
|
+
let latest;
|
|
88
|
+
for (const e of this.index.values()) {
|
|
89
|
+
if (!latest || e.timestamp > latest.timestamp)
|
|
90
|
+
latest = e;
|
|
91
|
+
}
|
|
92
|
+
return latest;
|
|
93
|
+
}
|
|
94
|
+
autoCheckpoint(rootDir, filePath) {
|
|
95
|
+
const abs = path.isAbsolute(filePath) ? filePath : path.join(rootDir, filePath);
|
|
96
|
+
const label = `auto-${Date.now()}`;
|
|
97
|
+
this.save([abs], label);
|
|
98
|
+
}
|
|
99
|
+
static instance_ = null;
|
|
100
|
+
static instance() {
|
|
101
|
+
if (!Checkpoint.instance_)
|
|
102
|
+
Checkpoint.instance_ = new Checkpoint();
|
|
103
|
+
return Checkpoint.instance_;
|
|
104
|
+
}
|
|
105
|
+
}
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
function fetchWithTimeout(url, init = {}, timeoutMs) {
|
|
2
|
+
const controller = new AbortController();
|
|
3
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
4
|
+
const signal = init.signal
|
|
5
|
+
? AbortSignal.any([init.signal, controller.signal])
|
|
6
|
+
: controller.signal;
|
|
7
|
+
return fetch(url, { ...init, signal }).finally(() => clearTimeout(timer));
|
|
8
|
+
}
|
|
9
|
+
export class FreeRouterClient {
|
|
10
|
+
baseURL;
|
|
11
|
+
timeoutMs;
|
|
12
|
+
retries;
|
|
13
|
+
constructor(baseURL, opts = {}) {
|
|
14
|
+
this.baseURL = (baseURL ?? opts.baseURL ?? "http://localhost:31415").replace(/\/$/, "");
|
|
15
|
+
this.timeoutMs = opts.timeoutMs ?? 30000;
|
|
16
|
+
this.retries = opts.retries ?? 3;
|
|
17
|
+
}
|
|
18
|
+
async request(path, init = {}, tryNextPort = false) {
|
|
19
|
+
let lastErr = null;
|
|
20
|
+
const ports = tryNextPort ? [31415, 31416, 31417, 31418] : [null];
|
|
21
|
+
for (const port of ports) {
|
|
22
|
+
const base = port ? `http://localhost:${port}` : this.baseURL;
|
|
23
|
+
const url = `${base}${path}`;
|
|
24
|
+
for (let attempt = 0; attempt <= this.retries; attempt++) {
|
|
25
|
+
try {
|
|
26
|
+
const res = await fetchWithTimeout(url, init, this.timeoutMs);
|
|
27
|
+
if (!res.ok) {
|
|
28
|
+
const text = await res.text().catch(() => "");
|
|
29
|
+
let body = {};
|
|
30
|
+
try {
|
|
31
|
+
body = text ? JSON.parse(text) : {};
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
body = { raw: text };
|
|
35
|
+
}
|
|
36
|
+
const msg = body?.error?.message || body?.error || `HTTP ${res.status}`;
|
|
37
|
+
throw new Error(`${msg} (status ${res.status})`);
|
|
38
|
+
}
|
|
39
|
+
return (await res.json());
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
lastErr = err;
|
|
43
|
+
// small backoff before retrying the same URL
|
|
44
|
+
if (attempt < this.retries) {
|
|
45
|
+
await new Promise((r) => setTimeout(r, 200 * (attempt + 1)));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
throw lastErr ?? new Error("Request failed");
|
|
51
|
+
}
|
|
52
|
+
async health() {
|
|
53
|
+
return this.request("/health");
|
|
54
|
+
}
|
|
55
|
+
async providers() {
|
|
56
|
+
return this.request("/providers");
|
|
57
|
+
}
|
|
58
|
+
async listModels() {
|
|
59
|
+
return this.request("/v1/models");
|
|
60
|
+
}
|
|
61
|
+
async chat(messages, opts) {
|
|
62
|
+
const body = {
|
|
63
|
+
model: opts?.model,
|
|
64
|
+
messages,
|
|
65
|
+
tools: opts?.tools ?? [],
|
|
66
|
+
temperature: opts?.temperature,
|
|
67
|
+
max_tokens: opts?.maxTokens,
|
|
68
|
+
};
|
|
69
|
+
if (opts?.stream)
|
|
70
|
+
body.stream = true;
|
|
71
|
+
return this.request("/v1/chat", {
|
|
72
|
+
method: "POST",
|
|
73
|
+
headers: { "Content-Type": "application/json" },
|
|
74
|
+
body: JSON.stringify(body),
|
|
75
|
+
}, true);
|
|
76
|
+
}
|
|
77
|
+
async chatCompletion(messages, opts) {
|
|
78
|
+
const body = {
|
|
79
|
+
model: opts?.model,
|
|
80
|
+
messages,
|
|
81
|
+
tools: opts?.tools ?? [],
|
|
82
|
+
temperature: opts?.temperature,
|
|
83
|
+
max_tokens: opts?.maxTokens,
|
|
84
|
+
stream: !!opts?.stream,
|
|
85
|
+
};
|
|
86
|
+
return this.request("/v1/chat/completions", {
|
|
87
|
+
method: "POST",
|
|
88
|
+
headers: { "Content-Type": "application/json" },
|
|
89
|
+
body: JSON.stringify(body),
|
|
90
|
+
}, true);
|
|
91
|
+
}
|
|
92
|
+
async resetHealth() {
|
|
93
|
+
return this.request("/reset-health", { method: "POST", headers: { "Content-Type": "application/json" } }, true);
|
|
94
|
+
}
|
|
95
|
+
}
|