codex-multi-auth 2.1.8 → 2.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +413 -411
  2. package/dist/lib/accounts.d.ts +2 -0
  3. package/dist/lib/accounts.d.ts.map +1 -1
  4. package/dist/lib/accounts.js +26 -2
  5. package/dist/lib/accounts.js.map +1 -1
  6. package/dist/lib/auth/device-auth.d.ts +1 -0
  7. package/dist/lib/auth/device-auth.d.ts.map +1 -1
  8. package/dist/lib/auth/device-auth.js +6 -2
  9. package/dist/lib/auth/device-auth.js.map +1 -1
  10. package/dist/lib/codex-manager/commands/forecast.d.ts +5 -1
  11. package/dist/lib/codex-manager/commands/forecast.d.ts.map +1 -1
  12. package/dist/lib/codex-manager/commands/forecast.js +14 -1
  13. package/dist/lib/codex-manager/commands/forecast.js.map +1 -1
  14. package/dist/lib/codex-manager/commands/report.d.ts +2 -0
  15. package/dist/lib/codex-manager/commands/report.d.ts.map +1 -1
  16. package/dist/lib/codex-manager/commands/report.js +17 -1
  17. package/dist/lib/codex-manager/commands/report.js.map +1 -1
  18. package/dist/lib/codex-manager/commands/rotation.d.ts +1 -1
  19. package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
  20. package/dist/lib/codex-manager/commands/rotation.js +71 -1
  21. package/dist/lib/codex-manager/commands/rotation.js.map +1 -1
  22. package/dist/lib/codex-manager/repair-commands.d.ts.map +1 -1
  23. package/dist/lib/codex-manager/repair-commands.js +35 -0
  24. package/dist/lib/codex-manager/repair-commands.js.map +1 -1
  25. package/dist/lib/codex-manager.d.ts.map +1 -1
  26. package/dist/lib/codex-manager.js +36 -2
  27. package/dist/lib/codex-manager.js.map +1 -1
  28. package/dist/lib/forecast.d.ts +9 -0
  29. package/dist/lib/forecast.d.ts.map +1 -1
  30. package/dist/lib/forecast.js +36 -0
  31. package/dist/lib/forecast.js.map +1 -1
  32. package/dist/lib/runtime/runtime-observability.d.ts +17 -0
  33. package/dist/lib/runtime/runtime-observability.d.ts.map +1 -1
  34. package/dist/lib/runtime/runtime-observability.js +48 -0
  35. package/dist/lib/runtime/runtime-observability.js.map +1 -1
  36. package/dist/lib/runtime-rotation-proxy.d.ts +1 -0
  37. package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
  38. package/dist/lib/runtime-rotation-proxy.js +119 -20
  39. package/dist/lib/runtime-rotation-proxy.js.map +1 -1
  40. package/package.json +193 -185
  41. package/scripts/audit-dev-allowlist.js +0 -220
  42. package/scripts/bench-format/codex-host.mjs +0 -275
  43. package/scripts/bench-format/hashline-v2.mjs +0 -642
  44. package/scripts/bench-format/models.mjs +0 -110
  45. package/scripts/bench-format/render.mjs +0 -496
  46. package/scripts/bench-format/stats.mjs +0 -54
  47. package/scripts/bench-format/tasks.mjs +0 -151
  48. package/scripts/benchmark-edit-formats.mjs +0 -1164
  49. package/scripts/benchmark-render-dashboard.mjs +0 -49
  50. package/scripts/benchmark-runtime-path.mjs +0 -166
  51. package/scripts/check-pack-budget-lib.js +0 -157
  52. package/scripts/check-pack-budget.mjs +0 -10
  53. package/scripts/copy-oauth-success.js +0 -76
  54. package/scripts/install-codex-auth.js +0 -258
  55. package/scripts/repo-hygiene.js +0 -320
  56. package/scripts/test-all-models.sh +0 -7
  57. package/scripts/test-model-matrix.js +0 -687
  58. package/scripts/update-vendor-provenance.mjs +0 -96
  59. package/scripts/validate-model-map.sh +0 -7
  60. package/scripts/verify-vendor-provenance.mjs +0 -53
@@ -1,110 +0,0 @@
1
- import { spawnSync } from "node:child_process";
2
- import { resolveCodexExecutable } from "./codex-host.mjs";
3
-
4
- const FALLBACK_OPENAI_CODEX_STABLE = [
5
- "openai/gpt-5.3-codex",
6
- "openai/gpt-5.4",
7
- "openai/gpt-5.4-mini",
8
- "openai/gpt-5.5",
9
- ];
10
-
11
- const OPENAI_CODEX_PREFIXES = ["openai/", "openai-multi/"];
12
-
13
- export function isOpenAiCodexProviderPrefix(modelId) {
14
- return OPENAI_CODEX_PREFIXES.some((prefix) => modelId.startsWith(prefix));
15
- }
16
-
17
- export function codexTail(modelId) {
18
- const slash = modelId.indexOf("/");
19
- return slash >= 0 ? modelId.slice(slash + 1) : modelId;
20
- }
21
-
22
- export function canonicalCodexModelId(modelId) {
23
- return `openai/${codexTail(modelId)}`;
24
- }
25
-
26
- export function aliasCandidatesForCodexModel(modelId) {
27
- if (!isOpenAiCodexProviderPrefix(modelId)) {
28
- return [modelId];
29
- }
30
- const tail = codexTail(modelId);
31
- const candidates = [`openai/${tail}`, `openai-multi/${tail}`];
32
- return [...new Set([tail, modelId, ...candidates])];
33
- }
34
-
35
- export function isStableOpenAiCodexModel(modelId) {
36
- if (!isOpenAiCodexProviderPrefix(modelId)) {
37
- return false;
38
- }
39
- if (!modelId.includes("codex")) {
40
- return false;
41
- }
42
- if (modelId.includes("-spark") || modelId.includes("-latest")) {
43
- return false;
44
- }
45
- return true;
46
- }
47
-
48
- export function listCodexModels() {
49
- const executable = resolveCodexExecutable();
50
- const child = spawnSync(executable.command, [...(executable.argsPrefix ?? []), "models"], {
51
- encoding: "utf8",
52
- windowsHide: true,
53
- shell: executable.shell,
54
- stdio: ["ignore", "pipe", "pipe"],
55
- maxBuffer: 10 * 1024 * 1024,
56
- timeout: Number.parseInt(process.env.CODEX_MODELS_TIMEOUT_MS ?? "30000", 10),
57
- killSignal: "SIGKILL",
58
- });
59
- if (child.error && child.error.code === "ETIMEDOUT") {
60
- throw new Error(`Timed out while listing Codex models after ${process.env.CODEX_MODELS_TIMEOUT_MS ?? "30000"}ms`);
61
- }
62
- const text = `${child.stdout ?? ""}\n${child.stderr ?? ""}`;
63
- if ((child.status ?? 1) !== 0) {
64
- throw new Error(`Failed to list Codex models (exit=${child.status ?? 1})`);
65
- }
66
- return text
67
- .split(/\r?\n/)
68
- .map((line) => line.trim())
69
- .filter((line) => line.length > 0)
70
- .filter((line) => !line.startsWith("Using Node "));
71
- }
72
-
73
- function dedupeCodexModels(models) {
74
- const byCanonical = new Map();
75
- for (const modelId of models) {
76
- const canonical = canonicalCodexModelId(modelId);
77
- const existing = byCanonical.get(canonical);
78
- if (!existing) {
79
- byCanonical.set(canonical, modelId);
80
- continue;
81
- }
82
- // Prefer the shorter/default provider form when both exist.
83
- if (existing.startsWith("openai-multi/") && modelId.startsWith("openai/")) {
84
- byCanonical.set(canonical, modelId);
85
- }
86
- }
87
- return [...byCanonical.values()].sort((a, b) => canonicalCodexModelId(a).localeCompare(canonicalCodexModelId(b)));
88
- }
89
-
90
- export function resolveModelPreset(presetName, explicitModels) {
91
- if (explicitModels && explicitModels.length > 0) {
92
- return explicitModels;
93
- }
94
-
95
- if (presetName !== "codex-core") {
96
- throw new Error(`Unsupported preset: ${presetName}`);
97
- }
98
-
99
- try {
100
- const models = dedupeCodexModels(listCodexModels().filter(isStableOpenAiCodexModel));
101
- if (models.length > 0) {
102
- return models;
103
- }
104
- } catch {
105
- // Fall back to static list.
106
- }
107
-
108
- return [...FALLBACK_OPENAI_CODEX_STABLE];
109
- }
110
-
@@ -1,496 +0,0 @@
1
- import { pctDelta, round1 } from "./stats.mjs";
2
-
3
- function formatNumber(value, digits = 1) {
4
- if (typeof value !== "number" || !Number.isFinite(value)) {
5
- return "-";
6
- }
7
- return value.toFixed(digits);
8
- }
9
-
10
- function formatMs(value) {
11
- if (typeof value !== "number" || !Number.isFinite(value)) {
12
- return "-";
13
- }
14
- if (value >= 1000) {
15
- return `${(value / 1000).toFixed(1)}s`;
16
- }
17
- return `${Math.round(value)}ms`;
18
- }
19
-
20
- function formatPct(value, digits = 1) {
21
- if (typeof value !== "number" || !Number.isFinite(value)) {
22
- return "-";
23
- }
24
- return `${value.toFixed(digits)}%`;
25
- }
26
-
27
- function formatSigned(value, digits = 1, suffix = "") {
28
- if (typeof value !== "number" || !Number.isFinite(value)) {
29
- return "-";
30
- }
31
- const sign = value > 0 ? "+" : "";
32
- return `${sign}${value.toFixed(digits)}${suffix}`;
33
- }
34
-
35
- function modeMetric(row, mode) {
36
- return row.modes?.[mode] ?? null;
37
- }
38
-
39
- function buildLeaderboardRows(summary) {
40
- const rows = summary.rows ?? [];
41
- return [...rows].sort((left, right) => {
42
- const lv2 = modeMetric(left, "hashline_v2")?.accuracyPct ?? -1;
43
- const rv2 = modeMetric(right, "hashline_v2")?.accuracyPct ?? -1;
44
- if (rv2 !== lv2) return rv2 - lv2;
45
- const lhash = modeMetric(left, "hashline")?.accuracyPct ?? -1;
46
- const rhash = modeMetric(right, "hashline")?.accuracyPct ?? -1;
47
- if (rhash !== lhash) return rhash - lhash;
48
- return (left.displayName ?? left.modelId).localeCompare(right.displayName ?? right.modelId);
49
- });
50
- }
51
-
52
- export function buildMarkdownReport(summary) {
53
- const rows = buildLeaderboardRows(summary);
54
- const lines = [];
55
- const write = (line = "") => lines.push(line);
56
-
57
- write("# Code Edit Format Benchmark");
58
- write("");
59
- write(`Generated: ${summary.meta?.generatedAt ?? "-"}`);
60
- write(`Preset: ${summary.meta?.preset ?? "-"}`);
61
- write(`Models: ${(summary.meta?.models ?? []).length}`);
62
- write(`Tasks: ${(summary.meta?.tasks ?? []).length}`);
63
- write(`Modes: ${(summary.meta?.modes ?? []).join(", ")}`);
64
- write(`Runs: ${summary.meta?.runCount ?? 0} measured + ${summary.meta?.warmupCount ?? 0} warmup`);
65
- write("");
66
-
67
- if ((summary.failures ?? []).length > 0) {
68
- write("## Failures");
69
- write("");
70
- for (const failure of summary.failures.slice(0, 20)) {
71
- write(`- ${failure.modelId} / ${failure.mode} / ${failure.taskId} / ${failure.phase}: ${failure.reason}`);
72
- }
73
- if (summary.failures.length > 20) {
74
- write(`- ... ${summary.failures.length - 20} more`);
75
- }
76
- write("");
77
- }
78
-
79
- write("## Leaderboard (Accuracy First)");
80
- write("");
81
- write("| # | Model | Patch | Replace | Hashline | Hashline v2 | Delta v2 vs Patch | Delta v2 vs Replace | Tokens v2 vs Replace |");
82
- write("|---|-------|------:|--------:|---------:|------------:|--------------:|----------------:|--------------------:|");
83
- rows.forEach((row, index) => {
84
- const patch = modeMetric(row, "patch");
85
- const replace = modeMetric(row, "replace");
86
- const hashline = modeMetric(row, "hashline");
87
- const hashlineV2 = modeMetric(row, "hashline_v2");
88
- const v2VsPatch = (hashlineV2 && patch) ? round1((hashlineV2.accuracyPct ?? 0) - (patch.accuracyPct ?? 0)) : null;
89
- const v2VsReplace = (hashlineV2 && replace) ? round1((hashlineV2.accuracyPct ?? 0) - (replace.accuracyPct ?? 0)) : null;
90
- const tokenDelta = (hashlineV2 && replace)
91
- ? round1(pctDelta(hashlineV2.tokensTotalP50 ?? NaN, replace.tokensTotalP50 ?? NaN))
92
- : null;
93
- write(
94
- `| ${index + 1} | ${row.displayName} | ${formatPct(patch?.accuracyPct)} | ${formatPct(replace?.accuracyPct)} | ${formatPct(hashline?.accuracyPct)} | ${formatPct(hashlineV2?.accuracyPct)} | ${formatSigned(v2VsPatch)} | ${formatSigned(v2VsReplace)} | ${formatSigned(tokenDelta, 1, "%")} |`,
95
- );
96
- });
97
- write("");
98
-
99
- write("## Per-Mode Timing (p50 wall)");
100
- write("");
101
- write("| Model | Patch | Replace | Hashline | Hashline v2 |");
102
- write("|-------|------:|--------:|---------:|------------:|");
103
- rows.forEach((row) => {
104
- write(
105
- `| ${row.displayName} | ${formatMs(modeMetric(row, "patch")?.wallMsP50)} | ${formatMs(modeMetric(row, "replace")?.wallMsP50)} | ${formatMs(modeMetric(row, "hashline")?.wallMsP50)} | ${formatMs(modeMetric(row, "hashline_v2")?.wallMsP50)} |`,
106
- );
107
- });
108
- write("");
109
-
110
- write("## Notes");
111
- write("");
112
- write("- Accuracy is pass rate across the exact same task IDs for each model/mode.");
113
- write("- Tokens column in dashboard defaults to selected mode vs Replace baseline.");
114
- write("- Tool names are normalized for analysis (`edit` and `apply_patch` are treated as edit-call family).\n");
115
-
116
- return lines.join("\n");
117
- }
118
-
119
- function escapeHtml(value) {
120
- return String(value)
121
- .replace(/&/g, "&")
122
- .replace(/</g, "&lt;")
123
- .replace(/>/g, "&gt;")
124
- .replace(/\"/g, "&quot;")
125
- .replace(/'/g, "&#39;");
126
- }
127
-
128
- export function renderDashboardHtml(summary) {
129
- const rows = buildLeaderboardRows(summary);
130
- const payload = {
131
- meta: summary.meta,
132
- rows,
133
- };
134
-
135
- return `<!doctype html>
136
- <html lang="en">
137
- <head>
138
- <meta charset="utf-8" />
139
- <meta name="viewport" content="width=device-width, initial-scale=1" />
140
- <title>Code Edit Format Benchmark</title>
141
- <style>
142
- :root {
143
- --bg: #070b18;
144
- --panel: #0b1123;
145
- --panel-2: #0f172b;
146
- --line: rgba(149, 167, 255, 0.16);
147
- --text: #ecf2ff;
148
- --muted: #9cb0d9;
149
- --green: #4cf6a0;
150
- --green-2: #28d98d;
151
- --blue: #8fb6ff;
152
- --chip: rgba(255, 255, 255, 0.05);
153
- --danger: #ff6b86;
154
- }
155
- * { box-sizing: border-box; }
156
- body {
157
- margin: 0;
158
- color: var(--text);
159
- background:
160
- radial-gradient(1000px 500px at 20% -10%, rgba(92, 128, 255, 0.16), transparent 60%),
161
- radial-gradient(900px 500px at 100% 0%, rgba(76, 246, 160, 0.10), transparent 55%),
162
- var(--bg);
163
- font-family: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Helvetica, Arial, sans-serif;
164
- }
165
- .container {
166
- max-width: 1200px;
167
- margin: 22px auto;
168
- padding: 18px;
169
- border: 1px solid var(--line);
170
- border-radius: 18px;
171
- background: linear-gradient(180deg, rgba(255,255,255,0.01), rgba(255,255,255,0));
172
- box-shadow: 0 20px 80px rgba(0, 0, 0, 0.35);
173
- }
174
- .header h1 {
175
- margin: 0;
176
- font-size: 2rem;
177
- letter-spacing: -0.03em;
178
- }
179
- .header .subtitle {
180
- margin-top: 8px;
181
- color: var(--muted);
182
- font-size: 0.95rem;
183
- }
184
- .header .legend {
185
- color: var(--muted);
186
- }
187
- .header .legend b { color: var(--green); font-weight: 700; }
188
- .meta {
189
- margin-top: 6px;
190
- font-size: 0.8rem;
191
- color: var(--muted);
192
- }
193
- .controls {
194
- margin-top: 18px;
195
- display: flex;
196
- align-items: center;
197
- gap: 10px;
198
- flex-wrap: wrap;
199
- }
200
- .controls .label {
201
- color: var(--muted);
202
- font-size: 0.9rem;
203
- }
204
- .button {
205
- border: 1px solid var(--line);
206
- background: rgba(255,255,255,0.02);
207
- color: var(--text);
208
- border-radius: 999px;
209
- padding: 8px 14px;
210
- font-size: 0.85rem;
211
- cursor: pointer;
212
- }
213
- .button.active {
214
- border-color: rgba(76, 246, 160, 0.7);
215
- box-shadow: inset 0 0 0 1px rgba(76, 246, 160, 0.18);
216
- background: rgba(76, 246, 160, 0.08);
217
- }
218
- .table {
219
- margin-top: 18px;
220
- border-top: 1px solid var(--line);
221
- }
222
- .header-row, .row {
223
- display: grid;
224
- grid-template-columns: 58px minmax(170px, 220px) minmax(380px, 1fr) 85px 85px 90px;
225
- gap: 12px;
226
- align-items: center;
227
- }
228
- .header-row {
229
- color: var(--muted);
230
- font-size: 0.78rem;
231
- padding: 12px 14px;
232
- }
233
- .row {
234
- margin-top: 8px;
235
- padding: 12px 14px;
236
- border: 1px solid var(--line);
237
- border-radius: 12px;
238
- background: linear-gradient(180deg, rgba(255,255,255,0.02), rgba(255,255,255,0.01));
239
- }
240
- .rank {
241
- color: var(--blue);
242
- font-weight: 700;
243
- text-align: right;
244
- }
245
- .model {
246
- font-weight: 600;
247
- line-height: 1.15;
248
- }
249
- .model small {
250
- display: block;
251
- margin-top: 4px;
252
- color: var(--muted);
253
- font-weight: 500;
254
- }
255
- .bars { display: grid; gap: 7px; }
256
- .bar-line {
257
- display: grid;
258
- grid-template-columns: 84px 1fr 74px;
259
- gap: 8px;
260
- align-items: center;
261
- }
262
- .bar-label {
263
- color: var(--muted);
264
- font-size: 0.75rem;
265
- text-transform: uppercase;
266
- letter-spacing: 0.06em;
267
- }
268
- .bar-track {
269
- position: relative;
270
- height: 22px;
271
- border-radius: 6px;
272
- border: 1px solid rgba(255,255,255,0.04);
273
- background: rgba(255,255,255,0.03);
274
- overflow: hidden;
275
- }
276
- .bar-fill {
277
- position: absolute;
278
- inset: 0 auto 0 0;
279
- width: 0%;
280
- background: linear-gradient(90deg, rgba(255,255,255,0.08), rgba(255,255,255,0.10));
281
- }
282
- .bar-fill.hashline, .bar-fill.hashline_v2 {
283
- background: linear-gradient(90deg, rgba(40, 217, 141, 0.50), rgba(76, 246, 160, 0.78));
284
- }
285
- .bar-fill.selected {
286
- box-shadow: 0 0 20px rgba(76, 246, 160, 0.22);
287
- }
288
- .bar-value {
289
- text-align: right;
290
- font-weight: 700;
291
- color: var(--text);
292
- font-variant-numeric: tabular-nums;
293
- }
294
- .badge {
295
- display: inline-flex;
296
- justify-content: center;
297
- align-items: center;
298
- min-width: 56px;
299
- padding: 4px 10px;
300
- border-radius: 999px;
301
- font-weight: 700;
302
- font-size: 0.85rem;
303
- border: 1px solid rgba(255,255,255,0.08);
304
- background: rgba(255,255,255,0.03);
305
- color: var(--muted);
306
- }
307
- .badge.pos {
308
- color: var(--green);
309
- border-color: rgba(76,246,160,0.24);
310
- background: rgba(76,246,160,0.08);
311
- }
312
- .badge.neg {
313
- color: var(--danger);
314
- border-color: rgba(255,107,134,0.22);
315
- background: rgba(255,107,134,0.07);
316
- }
317
- .badge.neutral { color: var(--muted); }
318
- .details {
319
- margin-top: 6px;
320
- color: var(--muted);
321
- font-size: 0.75rem;
322
- }
323
- @media (max-width: 980px) {
324
- .header-row { display: none; }
325
- .row {
326
- grid-template-columns: 42px 1fr;
327
- gap: 8px;
328
- }
329
- .row > .metric, .row > .token { grid-column: 2 / -1; }
330
- }
331
- </style>
332
- </head>
333
- <body>
334
- <div class="container">
335
- <div class="header">
336
- <h1>Code Edit Format Benchmark</h1>
337
- <div class="subtitle">
338
- <span class="legend">Patch · Replace · <b>Hashline</b> · <b>Hashline v2</b></span>
339
- <span> - accuracy across ${(summary.meta?.models ?? []).length} models</span>
340
- </div>
341
- <div class="meta">Preset: ${escapeHtml(summary.meta?.preset ?? "-")} | Tasks: ${(summary.meta?.tasks ?? []).length} | Modes: ${escapeHtml((summary.meta?.modes ?? []).join(", "))} | Generated: ${escapeHtml(summary.meta?.generatedAt ?? "-")}</div>
342
- </div>
343
- <div class="controls">
344
- <span class="label">Sort</span>
345
- <button class="button active" data-sort="deltaVsReplaceHashline">Delta vs Replace</button>
346
- <button class="button" data-sort="deltaVsPatchHashline">Delta vs Patch</button>
347
- <button class="button" data-sort="hashlinePct">Hashline %</button>
348
- <button class="button" data-sort="hashlineV2Pct">Hashline v2 %</button>
349
- </div>
350
- <div class="table">
351
- <div class="header-row">
352
- <div></div>
353
- <div>MODEL</div>
354
- <div></div>
355
- <div>Delta Patch</div>
356
- <div>Delta Repl.</div>
357
- <div>TOKENS</div>
358
- </div>
359
- <div id="rows"></div>
360
- </div>
361
- </div>
362
- <script>
363
- const data = ${JSON.stringify(payload)};
364
-
365
- function round1(v) {
366
- return typeof v === 'number' && Number.isFinite(v) ? Math.round(v * 10) / 10 : null;
367
- }
368
-
369
- function pct(v) {
370
- return typeof v === 'number' && Number.isFinite(v) ? v.toFixed(1) + '%' : '-';
371
- }
372
-
373
- function badge(v, suffix = '') {
374
- if (typeof v !== 'number' || !Number.isFinite(v)) {
375
- return '<span class="badge neutral">-</span>';
376
- }
377
- const cls = v > 0 ? 'pos' : v < 0 ? 'neg' : 'neutral';
378
- const sign = v > 0 ? '+' : '';
379
- return '<span class="badge ' + cls + '">' + sign + v.toFixed(1) + suffix + '</span>';
380
- }
381
-
382
- function mode(row, name) {
383
- return (row.modes && row.modes[name]) || {};
384
- }
385
-
386
- function tokenDeltaPct(selected, replace) {
387
- if (!selected || !replace) return null;
388
- const a = selected.tokensTotalP50;
389
- const b = replace.tokensTotalP50;
390
- if (!Number.isFinite(a) || !Number.isFinite(b) || b === 0) return null;
391
- return ((a - b) / b) * 100;
392
- }
393
-
394
- function toRankData(rows) {
395
- return rows.map((row) => {
396
- const patch = mode(row, 'patch');
397
- const replace = mode(row, 'replace');
398
- const hashline = mode(row, 'hashline');
399
- const hashlineV2 = mode(row, 'hashline_v2');
400
- return {
401
- ...row,
402
- _metrics: {
403
- patchPct: patch.accuracyPct ?? -1,
404
- replacePct: replace.accuracyPct ?? -1,
405
- hashlinePct: hashline.accuracyPct ?? -1,
406
- hashlineV2Pct: hashlineV2.accuracyPct ?? -1,
407
- deltaVsReplaceHashline: Number.isFinite(hashline.accuracyPct) && Number.isFinite(replace.accuracyPct)
408
- ? hashline.accuracyPct - replace.accuracyPct
409
- : -999,
410
- deltaVsPatchHashline: Number.isFinite(hashline.accuracyPct) && Number.isFinite(patch.accuracyPct)
411
- ? hashline.accuracyPct - patch.accuracyPct
412
- : -999,
413
- },
414
- };
415
- });
416
- }
417
-
418
- const rankedRows = toRankData(data.rows || []);
419
- let activeSort = 'deltaVsReplaceHashline';
420
-
421
- function sortRows(rows, key) {
422
- return [...rows].sort((a, b) => {
423
- const diff = (b._metrics[key] ?? -999) - (a._metrics[key] ?? -999);
424
- if (diff !== 0) return diff;
425
- const av2 = b._metrics.hashlineV2Pct - a._metrics.hashlineV2Pct;
426
- if (av2 !== 0) return av2;
427
- return (a.displayName || a.modelId).localeCompare(b.displayName || b.modelId);
428
- });
429
- }
430
-
431
- function createBarLine(label, metric, selectedKey) {
432
- const pctValue = metric && Number.isFinite(metric.accuracyPct) ? metric.accuracyPct : 0;
433
- const modeName = (label === 'PATCH' ? 'patch' : label === 'REPLACE' ? 'replace' : label === 'HASHLINE' ? 'hashline' : 'hashline_v2');
434
- const selected = selectedKey === 'hashlinePct' ? modeName === 'hashline' : selectedKey === 'hashlineV2Pct' ? modeName === 'hashline_v2' : modeName === 'hashline';
435
- return ''
436
- + '<div class="bar-line">'
437
- + '<div class="bar-label">' + label + '</div>'
438
- + '<div class="bar-track"><div class="bar-fill ' + modeName + ' ' + (selected ? 'selected' : '') + '" style="width:' + Math.max(0, Math.min(100, pctValue)) + '%"></div></div>'
439
- + '<div class="bar-value">' + pct(metric ? metric.accuracyPct : null) + '</div>'
440
- + '</div>';
441
- }
442
-
443
- function render() {
444
- const rowsEl = document.getElementById('rows');
445
- const rows = sortRows(rankedRows, activeSort);
446
- rowsEl.innerHTML = rows.map((row, index) => {
447
- const patch = mode(row, 'patch');
448
- const replace = mode(row, 'replace');
449
- const hashline = mode(row, 'hashline');
450
- const hashlineV2 = mode(row, 'hashline_v2');
451
-
452
- const selectedMode = activeSort === 'hashlineV2Pct' ? hashlineV2 : hashline;
453
- const deltaPatch = selectedMode && patch && Number.isFinite(selectedMode.accuracyPct) && Number.isFinite(patch.accuracyPct)
454
- ? round1(selectedMode.accuracyPct - patch.accuracyPct)
455
- : null;
456
- const deltaReplace = selectedMode && replace && Number.isFinite(selectedMode.accuracyPct) && Number.isFinite(replace.accuracyPct)
457
- ? round1(selectedMode.accuracyPct - replace.accuracyPct)
458
- : null;
459
- const tokensDelta = tokenDeltaPct(selectedMode, replace);
460
-
461
- const p50V2 = patch && hashlineV2 ? (Number.isFinite(hashlineV2.wallMsP50) ? (hashlineV2.wallMsP50 / 1000).toFixed(1) + 's' : '-') : '-';
462
- return ''
463
- + '<div class="row">'
464
- + '<div class="rank">' + (index + 1) + '</div>'
465
- + '<div class="model">' + (row.displayName || row.modelId) + '<small>' + row.modelId + '</small></div>'
466
- + '<div>'
467
- + '<div class="bars">'
468
- + createBarLine('PATCH', patch, activeSort)
469
- + createBarLine('REPLACE', replace, activeSort)
470
- + createBarLine('HASHLINE', hashline, activeSort)
471
- + createBarLine('HASHLINE v2', hashlineV2, activeSort)
472
- + '</div>'
473
- + '<div class="details">Task pass rate across identical task IDs and measured runs. p50 wall (hashline v2): ' + p50V2 + '</div>'
474
- + '</div>'
475
- + '<div class="metric">' + badge(deltaPatch) + '</div>'
476
- + '<div class="metric">' + badge(deltaReplace) + '</div>'
477
- + '<div class="token">' + badge(tokensDelta, '%') + '</div>'
478
- + '</div>';
479
- }).join('');
480
- }
481
-
482
- for (const button of document.querySelectorAll('.button[data-sort]')) {
483
- button.addEventListener('click', () => {
484
- activeSort = button.dataset.sort;
485
- for (const other of document.querySelectorAll('.button[data-sort]')) {
486
- other.classList.toggle('active', other === button);
487
- }
488
- render();
489
- });
490
- }
491
-
492
- render();
493
- </script>
494
- </body>
495
- </html>`;
496
- }
@@ -1,54 +0,0 @@
1
- export function percentile(values, p) {
2
- if (!Array.isArray(values) || values.length === 0) {
3
- return null;
4
- }
5
- const sorted = [...values].sort((a, b) => a - b);
6
- if (sorted.length === 1) {
7
- return sorted[0];
8
- }
9
- const index = (sorted.length - 1) * p;
10
- const low = Math.floor(index);
11
- const high = Math.ceil(index);
12
- if (low === high) {
13
- return sorted[low];
14
- }
15
- const frac = index - low;
16
- return sorted[low] + (sorted[high] - sorted[low]) * frac;
17
- }
18
-
19
- export function stats(values) {
20
- const nums = values.filter((value) => typeof value === "number" && Number.isFinite(value));
21
- if (nums.length === 0) {
22
- return null;
23
- }
24
- const sum = nums.reduce((acc, value) => acc + value, 0);
25
- return {
26
- n: nums.length,
27
- min: Math.min(...nums),
28
- p50: percentile(nums, 0.5),
29
- p95: percentile(nums, 0.95),
30
- max: Math.max(...nums),
31
- mean: sum / nums.length,
32
- };
33
- }
34
-
35
- export function safePercent(numerator, denominator) {
36
- if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator <= 0) {
37
- return null;
38
- }
39
- return (numerator / denominator) * 100;
40
- }
41
-
42
- export function round1(value) {
43
- if (typeof value !== "number" || !Number.isFinite(value)) {
44
- return null;
45
- }
46
- return Math.round(value * 10) / 10;
47
- }
48
-
49
- export function pctDelta(current, baseline) {
50
- if (!Number.isFinite(current) || !Number.isFinite(baseline) || baseline === 0) {
51
- return null;
52
- }
53
- return ((current - baseline) / baseline) * 100;
54
- }