ai-speedometer 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-speedometer +488 -38
- package/package.json +1 -1
package/dist/ai-speedometer
CHANGED
|
@@ -1932,6 +1932,7 @@ async function benchmarkSingleModelRest(model, logger) {
|
|
|
1932
1932
|
const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
|
|
1933
1933
|
const totalTokens = finalInputTokens + finalOutputTokens;
|
|
1934
1934
|
const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
|
|
1935
|
+
const f1000 = tokensPerSecond > 0 ? 1000 * (timeToFirstToken / 1000 + 300 / tokensPerSecond) / 3600 : Infinity;
|
|
1935
1936
|
return {
|
|
1936
1937
|
model: model.name,
|
|
1937
1938
|
provider: model.providerName,
|
|
@@ -1939,6 +1940,7 @@ async function benchmarkSingleModelRest(model, logger) {
|
|
|
1939
1940
|
timeToFirstToken,
|
|
1940
1941
|
tokenCount: finalOutputTokens,
|
|
1941
1942
|
tokensPerSecond,
|
|
1943
|
+
f1000,
|
|
1942
1944
|
promptTokens: finalInputTokens,
|
|
1943
1945
|
totalTokens,
|
|
1944
1946
|
usedEstimateForOutput,
|
|
@@ -1954,6 +1956,7 @@ async function benchmarkSingleModelRest(model, logger) {
|
|
|
1954
1956
|
timeToFirstToken: 0,
|
|
1955
1957
|
tokenCount: 0,
|
|
1956
1958
|
tokensPerSecond: 0,
|
|
1959
|
+
f1000: Infinity,
|
|
1957
1960
|
promptTokens: 0,
|
|
1958
1961
|
totalTokens: 0,
|
|
1959
1962
|
usedEstimateForOutput: true,
|
|
@@ -2019,6 +2022,8 @@ function buildJsonOutput(providerName, providerId, modelName, modelId, result, f
|
|
|
2019
2022
|
timeToFirstToken: result.timeToFirstToken,
|
|
2020
2023
|
timeToFirstTokenSeconds: result.timeToFirstToken / 1000,
|
|
2021
2024
|
tokensPerSecond: result.tokensPerSecond,
|
|
2025
|
+
f1000: result.f1000,
|
|
2026
|
+
f1000Hours: result.f1000 === Infinity ? null : result.f1000,
|
|
2022
2027
|
outputTokens: result.tokenCount,
|
|
2023
2028
|
promptTokens: result.promptTokens,
|
|
2024
2029
|
totalTokens: result.totalTokens,
|
|
@@ -3011,7 +3016,7 @@ var package_default;
|
|
|
3011
3016
|
var init_package = __esm(() => {
|
|
3012
3017
|
package_default = {
|
|
3013
3018
|
name: "ai-speedometer",
|
|
3014
|
-
version: "2.
|
|
3019
|
+
version: "2.3.0",
|
|
3015
3020
|
description: "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
|
|
3016
3021
|
bin: {
|
|
3017
3022
|
"ai-speedometer": "dist/ai-speedometer",
|
|
@@ -3155,6 +3160,7 @@ function MainMenuScreen() {
|
|
|
3155
3160
|
const ITEMS = [
|
|
3156
3161
|
{ label: "\u26A1 Run Benchmark", desc: "test model speed & throughput", color: theme.accent },
|
|
3157
3162
|
{ label: "\u2699 Manage Models", desc: "add providers and configure", color: theme.secondary },
|
|
3163
|
+
{ label: "? FAQ / Learn", desc: "how metrics work & resources", color: theme.primary },
|
|
3158
3164
|
{ label: "\u2715 Exit", desc: "quit the application", color: theme.error }
|
|
3159
3165
|
];
|
|
3160
3166
|
useAppKeyboard((key) => {
|
|
@@ -3168,6 +3174,8 @@ function MainMenuScreen() {
|
|
|
3168
3174
|
else if (cursor === 1)
|
|
3169
3175
|
navigate("model-menu");
|
|
3170
3176
|
else if (cursor === 2)
|
|
3177
|
+
navigate("faq");
|
|
3178
|
+
else if (cursor === 3)
|
|
3171
3179
|
renderer.destroy();
|
|
3172
3180
|
}
|
|
3173
3181
|
});
|
|
@@ -3202,7 +3210,7 @@ function MainMenuScreen() {
|
|
|
3202
3210
|
borderStyle: "rounded",
|
|
3203
3211
|
borderColor: theme.border,
|
|
3204
3212
|
backgroundColor: theme.background,
|
|
3205
|
-
width:
|
|
3213
|
+
width: 48,
|
|
3206
3214
|
children: ITEMS.map((item, i) => {
|
|
3207
3215
|
const active = i === cursor;
|
|
3208
3216
|
return /* @__PURE__ */ jsxDEV7("box", {
|
|
@@ -3712,8 +3720,9 @@ var init_ModelSelectScreen = __esm(() => {
|
|
|
3712
3720
|
|
|
3713
3721
|
// src/tui/components/BarChart.tsx
|
|
3714
3722
|
import { jsxDEV as jsxDEV11 } from "@opentui/react/jsx-dev-runtime";
|
|
3715
|
-
function BarChart({ value, max, width, color }) {
|
|
3716
|
-
const
|
|
3723
|
+
function BarChart({ value, max, width, color, inverted = false }) {
|
|
3724
|
+
const normalizedValue = inverted && max > 0 ? Math.max(0, max - value) : value;
|
|
3725
|
+
const filled = max === 0 ? 0 : Math.round(normalizedValue / max * width);
|
|
3717
3726
|
const empty = width - filled;
|
|
3718
3727
|
const bar = "\u2588".repeat(filled) + "\u2591".repeat(empty);
|
|
3719
3728
|
return /* @__PURE__ */ jsxDEV11("text", {
|
|
@@ -3817,13 +3826,17 @@ function trunc(s, w) {
|
|
|
3817
3826
|
function ResultsTable({ results, pendingCount }) {
|
|
3818
3827
|
const theme = useTheme();
|
|
3819
3828
|
const sorted = [...results].sort((a, b) => b.tokensPerSecond - a.tokensPerSecond);
|
|
3820
|
-
const C = { rank: 4, model:
|
|
3821
|
-
const totalW = C.rank + C.model + C.prov + C.time + C.ttft + C.tps + C.out + C.inp + C.tot +
|
|
3829
|
+
const C = { rank: 4, model: 16, prov: 10, time: 8, ttft: 7, tps: 9, f1000: 8, out: 6, inp: 6, tot: 6 };
|
|
3830
|
+
const totalW = C.rank + C.model + C.prov + C.time + C.ttft + C.tps + C.f1000 + C.out + C.inp + C.tot + 10;
|
|
3822
3831
|
const sep = "\u2500".repeat(totalW);
|
|
3823
|
-
|
|
3824
|
-
|
|
3832
|
+
const maxTps = Math.max(...results.map((r) => r.tokensPerSecond), 0);
|
|
3833
|
+
const minTtft = Math.min(...results.map((r) => r.timeToFirstToken), Infinity);
|
|
3834
|
+
const validF1000s = results.map((r) => r.f1000).filter((f) => f !== Infinity);
|
|
3835
|
+
const minF1000 = validF1000s.length > 0 ? Math.min(...validF1000s) : Infinity;
|
|
3836
|
+
function row(rank, model, prov, time, ttft, tps, f1000, out, inp, tot) {
|
|
3837
|
+
return lpad(rank, C.rank) + " \u2502 " + rpad(model, C.model) + " \u2502 " + rpad(prov, C.prov) + " \u2502 " + lpad(time, C.time) + " \u2502 " + lpad(ttft, C.ttft) + " \u2502 " + lpad(tps, C.tps) + " \u2502 " + lpad(f1000, C.f1000) + " \u2502 " + lpad(out, C.out) + " \u2502 " + lpad(inp, C.inp) + " \u2502 " + lpad(tot, C.tot);
|
|
3825
3838
|
}
|
|
3826
|
-
const header = row("#", "Model", "Provider", "Time(s)", "TTFT(s)", "
|
|
3839
|
+
const header = row("#", "Model", "Provider", "Time(s)", "TTFT(s)", "Tok/s", "F1000(h)", "Out", "In", "Total");
|
|
3827
3840
|
return /* @__PURE__ */ jsxDEV13("box", {
|
|
3828
3841
|
flexDirection: "column",
|
|
3829
3842
|
paddingLeft: 1,
|
|
@@ -3845,21 +3858,87 @@ function ResultsTable({ results, pendingCount }) {
|
|
|
3845
3858
|
}, undefined, false, undefined, this),
|
|
3846
3859
|
sorted.map((r, i) => {
|
|
3847
3860
|
const rank = `${i + 1}`;
|
|
3848
|
-
const timeSec = (r.totalTime / 1000).toFixed(
|
|
3861
|
+
const timeSec = (r.totalTime / 1000).toFixed(1);
|
|
3849
3862
|
const ttftSec = (r.timeToFirstToken / 1000).toFixed(2);
|
|
3850
|
-
const tps = r.tokensPerSecond.toFixed(
|
|
3851
|
-
const
|
|
3852
|
-
const
|
|
3853
|
-
const
|
|
3863
|
+
const tps = r.tokensPerSecond.toFixed(0);
|
|
3864
|
+
const f1000Val = r.f1000 === Infinity ? "\u221E" : r.f1000.toFixed(2);
|
|
3865
|
+
const outTok = r.tokenCount.toString() + (r.usedEstimateForOutput ? "*" : "");
|
|
3866
|
+
const inTok = r.promptTokens.toString() + (r.usedEstimateForInput ? "*" : "");
|
|
3867
|
+
const totTok = r.totalTokens.toString() + (r.usedEstimateForOutput || r.usedEstimateForInput ? "*" : "");
|
|
3854
3868
|
const hasEst = r.usedEstimateForOutput || r.usedEstimateForInput;
|
|
3855
|
-
const
|
|
3869
|
+
const isBestTps = r.tokensPerSecond === maxTps && maxTps > 0;
|
|
3870
|
+
const isBestTtft = r.timeToFirstToken === minTtft;
|
|
3871
|
+
const isBestF1000 = r.f1000 === minF1000 && r.f1000 !== Infinity;
|
|
3856
3872
|
return /* @__PURE__ */ jsxDEV13("box", {
|
|
3857
3873
|
height: 1,
|
|
3858
3874
|
flexDirection: "row",
|
|
3859
3875
|
children: [
|
|
3876
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3877
|
+
fg: theme.dim,
|
|
3878
|
+
children: [
|
|
3879
|
+
lpad(rank, C.rank),
|
|
3880
|
+
" \u2502 "
|
|
3881
|
+
]
|
|
3882
|
+
}, undefined, true, undefined, this),
|
|
3860
3883
|
/* @__PURE__ */ jsxDEV13("text", {
|
|
3861
3884
|
fg: theme.text,
|
|
3862
|
-
children:
|
|
3885
|
+
children: [
|
|
3886
|
+
rpad(trunc(r.model, C.model), C.model),
|
|
3887
|
+
" \u2502 "
|
|
3888
|
+
]
|
|
3889
|
+
}, undefined, true, undefined, this),
|
|
3890
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3891
|
+
fg: theme.dim,
|
|
3892
|
+
children: [
|
|
3893
|
+
rpad(trunc(r.provider, C.prov), C.prov),
|
|
3894
|
+
" \u2502 "
|
|
3895
|
+
]
|
|
3896
|
+
}, undefined, true, undefined, this),
|
|
3897
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3898
|
+
fg: theme.dim,
|
|
3899
|
+
children: [
|
|
3900
|
+
lpad(timeSec, C.time),
|
|
3901
|
+
" \u2502 "
|
|
3902
|
+
]
|
|
3903
|
+
}, undefined, true, undefined, this),
|
|
3904
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3905
|
+
fg: isBestTtft ? theme.success : theme.dim,
|
|
3906
|
+
children: [
|
|
3907
|
+
lpad(ttftSec, C.ttft),
|
|
3908
|
+
" \u2502 "
|
|
3909
|
+
]
|
|
3910
|
+
}, undefined, true, undefined, this),
|
|
3911
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3912
|
+
fg: isBestTps ? theme.success : theme.dim,
|
|
3913
|
+
children: [
|
|
3914
|
+
lpad(tps, C.tps),
|
|
3915
|
+
" \u2502 "
|
|
3916
|
+
]
|
|
3917
|
+
}, undefined, true, undefined, this),
|
|
3918
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3919
|
+
fg: isBestF1000 ? theme.success : theme.dim,
|
|
3920
|
+
children: [
|
|
3921
|
+
lpad(f1000Val, C.f1000),
|
|
3922
|
+
" \u2502 "
|
|
3923
|
+
]
|
|
3924
|
+
}, undefined, true, undefined, this),
|
|
3925
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3926
|
+
fg: theme.dim,
|
|
3927
|
+
children: [
|
|
3928
|
+
lpad(outTok, C.out),
|
|
3929
|
+
" \u2502 "
|
|
3930
|
+
]
|
|
3931
|
+
}, undefined, true, undefined, this),
|
|
3932
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3933
|
+
fg: theme.dim,
|
|
3934
|
+
children: [
|
|
3935
|
+
lpad(inTok, C.inp),
|
|
3936
|
+
" \u2502 "
|
|
3937
|
+
]
|
|
3938
|
+
}, undefined, true, undefined, this),
|
|
3939
|
+
/* @__PURE__ */ jsxDEV13("text", {
|
|
3940
|
+
fg: theme.dim,
|
|
3941
|
+
children: lpad(totTok, C.tot)
|
|
3863
3942
|
}, undefined, false, undefined, this),
|
|
3864
3943
|
hasEst && /* @__PURE__ */ jsxDEV13("text", {
|
|
3865
3944
|
fg: theme.warning,
|
|
@@ -3975,7 +4054,9 @@ function BenchmarkScreen() {
|
|
|
3975
4054
|
const maxTtft = Math.max(...done.map((m) => (m.result?.timeToFirstToken ?? 0) / 1000), 1);
|
|
3976
4055
|
const tpsRanked = done.slice().sort((a, b) => (b.result?.tokensPerSecond ?? 0) - (a.result?.tokensPerSecond ?? 0));
|
|
3977
4056
|
const ttftRanked = done.slice().sort((a, b) => (a.result?.timeToFirstToken ?? 0) - (b.result?.timeToFirstToken ?? 0));
|
|
4057
|
+
const f1000Ranked = done.slice().sort((a, b) => (a.result?.f1000 ?? Infinity) - (b.result?.f1000 ?? Infinity));
|
|
3978
4058
|
const maxTtftForBar = Math.max(...done.map((m) => (m.result?.timeToFirstToken ?? 0) / 1000), 1);
|
|
4059
|
+
const maxF1000 = Math.max(...done.map((m) => m.result?.f1000 ?? 0), 1);
|
|
3979
4060
|
const doneResults = tpsRanked.map((m) => m.result);
|
|
3980
4061
|
const pendingCount = running.length + pending.length;
|
|
3981
4062
|
const allRows = useMemo(() => {
|
|
@@ -4217,6 +4298,103 @@ function BenchmarkScreen() {
|
|
|
4217
4298
|
}, `ttft-${s.model.id}-${s.model.providerId}`, true, undefined, this));
|
|
4218
4299
|
}
|
|
4219
4300
|
}
|
|
4301
|
+
if (f1000Ranked.length > 0) {
|
|
4302
|
+
rows.push(/* @__PURE__ */ jsxDEV14("box", {
|
|
4303
|
+
height: 1,
|
|
4304
|
+
backgroundColor: theme.border
|
|
4305
|
+
}, "div-f1000", false, undefined, this));
|
|
4306
|
+
rows.push(/* @__PURE__ */ jsxDEV14("box", {
|
|
4307
|
+
height: 1,
|
|
4308
|
+
flexDirection: "row",
|
|
4309
|
+
paddingLeft: 1,
|
|
4310
|
+
children: /* @__PURE__ */ jsxDEV14("text", {
|
|
4311
|
+
fg: theme.primary,
|
|
4312
|
+
children: " F1000 RANKING - First to 1000 Requests (lower is better) "
|
|
4313
|
+
}, undefined, false, undefined, this)
|
|
4314
|
+
}, "hdr-f1000", false, undefined, this));
|
|
4315
|
+
for (const [i, s] of f1000Ranked.entries()) {
|
|
4316
|
+
const rank = i + 1;
|
|
4317
|
+
const rankFg = rank === 1 ? theme.accent : rank === 2 ? theme.secondary : theme.dim;
|
|
4318
|
+
const f1000 = s.result?.f1000 ?? Infinity;
|
|
4319
|
+
const f1000Str = f1000 === Infinity ? "\u221E" : f1000.toFixed(2);
|
|
4320
|
+
const ttft = (s.result?.timeToFirstToken ?? 0) / 1000;
|
|
4321
|
+
const tps = s.result?.tokensPerSecond ?? 0;
|
|
4322
|
+
const badge = rankBadge(rank).padStart(3);
|
|
4323
|
+
const modelCol = s.model.name.padEnd(18).slice(0, 18);
|
|
4324
|
+
const provCol = s.model.providerName.padEnd(12).slice(0, 12);
|
|
4325
|
+
rows.push(/* @__PURE__ */ jsxDEV14("box", {
|
|
4326
|
+
height: 1,
|
|
4327
|
+
flexDirection: "row",
|
|
4328
|
+
paddingLeft: 2,
|
|
4329
|
+
children: [
|
|
4330
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4331
|
+
fg: rankFg,
|
|
4332
|
+
children: [
|
|
4333
|
+
badge,
|
|
4334
|
+
" "
|
|
4335
|
+
]
|
|
4336
|
+
}, undefined, true, undefined, this),
|
|
4337
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4338
|
+
fg: theme.dim,
|
|
4339
|
+
children: " \u2502 "
|
|
4340
|
+
}, undefined, false, undefined, this),
|
|
4341
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4342
|
+
fg: theme.primary,
|
|
4343
|
+
children: [
|
|
4344
|
+
f1000Str.padStart(7),
|
|
4345
|
+
"h "
|
|
4346
|
+
]
|
|
4347
|
+
}, undefined, true, undefined, this),
|
|
4348
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4349
|
+
fg: theme.dim,
|
|
4350
|
+
children: " \u2502 "
|
|
4351
|
+
}, undefined, false, undefined, this),
|
|
4352
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4353
|
+
fg: theme.secondary,
|
|
4354
|
+
children: [
|
|
4355
|
+
ttft.toFixed(2).padStart(5),
|
|
4356
|
+
"s "
|
|
4357
|
+
]
|
|
4358
|
+
}, undefined, true, undefined, this),
|
|
4359
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4360
|
+
fg: theme.dim,
|
|
4361
|
+
children: " \u2502 "
|
|
4362
|
+
}, undefined, false, undefined, this),
|
|
4363
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4364
|
+
fg: theme.accent,
|
|
4365
|
+
children: [
|
|
4366
|
+
tps.toFixed(0).padStart(5),
|
|
4367
|
+
" tok/s "
|
|
4368
|
+
]
|
|
4369
|
+
}, undefined, true, undefined, this),
|
|
4370
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4371
|
+
fg: theme.dim,
|
|
4372
|
+
children: " \u2502 "
|
|
4373
|
+
}, undefined, false, undefined, this),
|
|
4374
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4375
|
+
fg: theme.text,
|
|
4376
|
+
children: [
|
|
4377
|
+
modelCol,
|
|
4378
|
+
" "
|
|
4379
|
+
]
|
|
4380
|
+
}, undefined, true, undefined, this),
|
|
4381
|
+
/* @__PURE__ */ jsxDEV14("text", {
|
|
4382
|
+
fg: theme.dim,
|
|
4383
|
+
children: [
|
|
4384
|
+
provCol,
|
|
4385
|
+
" \u2502 "
|
|
4386
|
+
]
|
|
4387
|
+
}, undefined, true, undefined, this),
|
|
4388
|
+
/* @__PURE__ */ jsxDEV14(BarChart, {
|
|
4389
|
+
value: f1000 === Infinity ? maxF1000 : f1000,
|
|
4390
|
+
max: maxF1000,
|
|
4391
|
+
width: BAR_W2,
|
|
4392
|
+
color: theme.primary
|
|
4393
|
+
}, undefined, false, undefined, this)
|
|
4394
|
+
]
|
|
4395
|
+
}, `f1000-${s.model.id}-${s.model.providerId}`, true, undefined, this));
|
|
4396
|
+
}
|
|
4397
|
+
}
|
|
4220
4398
|
if (allDone && errors.length > 0) {
|
|
4221
4399
|
rows.push(/* @__PURE__ */ jsxDEV14("box", {
|
|
4222
4400
|
height: 1,
|
|
@@ -4311,7 +4489,7 @@ function BenchmarkScreen() {
|
|
|
4311
4489
|
}, "results-empty", false, undefined, this));
|
|
4312
4490
|
}
|
|
4313
4491
|
return rows;
|
|
4314
|
-
}, [modelStates, allDone, tpsRanked, ttftRanked, doneResults, pendingCount, maxTps, maxTtftForBar, theme]);
|
|
4492
|
+
}, [modelStates, allDone, tpsRanked, ttftRanked, f1000Ranked, doneResults, pendingCount, maxTps, maxTtftForBar, maxF1000, theme]);
|
|
4315
4493
|
useAppKeyboard((key) => {
|
|
4316
4494
|
if (!allDone)
|
|
4317
4495
|
return;
|
|
@@ -5933,9 +6111,276 @@ var init_ListProvidersScreen = __esm(() => {
|
|
|
5933
6111
|
init_ThemeContext();
|
|
5934
6112
|
});
|
|
5935
6113
|
|
|
6114
|
+
// src/tui/screens/FAQScreen.tsx
|
|
6115
|
+
import { jsxDEV as jsxDEV19 } from "@opentui/react/jsx-dev-runtime";
|
|
6116
|
+
function FAQScreen() {
|
|
6117
|
+
const navigate = useNavigate();
|
|
6118
|
+
const theme = useTheme();
|
|
6119
|
+
useAppKeyboard((key) => {
|
|
6120
|
+
if (key.name === "escape" || key.name === "q") {
|
|
6121
|
+
navigate("main-menu");
|
|
6122
|
+
}
|
|
6123
|
+
});
|
|
6124
|
+
return /* @__PURE__ */ jsxDEV19("box", {
|
|
6125
|
+
flexDirection: "column",
|
|
6126
|
+
flexGrow: 1,
|
|
6127
|
+
alignItems: "center",
|
|
6128
|
+
padding: 1,
|
|
6129
|
+
children: /* @__PURE__ */ jsxDEV19("box", {
|
|
6130
|
+
flexDirection: "column",
|
|
6131
|
+
width: 70,
|
|
6132
|
+
children: [
|
|
6133
|
+
/* @__PURE__ */ jsxDEV19("box", {
|
|
6134
|
+
marginBottom: 1,
|
|
6135
|
+
children: /* @__PURE__ */ jsxDEV19("text", {
|
|
6136
|
+
fg: theme.primary,
|
|
6137
|
+
bold: true,
|
|
6138
|
+
children: "FAQ / Learn"
|
|
6139
|
+
}, undefined, false, undefined, this)
|
|
6140
|
+
}, undefined, false, undefined, this),
|
|
6141
|
+
/* @__PURE__ */ jsxDEV19("scrollbox", {
|
|
6142
|
+
focused: true,
|
|
6143
|
+
flexGrow: 1,
|
|
6144
|
+
children: /* @__PURE__ */ jsxDEV19("box", {
|
|
6145
|
+
flexDirection: "column",
|
|
6146
|
+
children: [
|
|
6147
|
+
/* @__PURE__ */ jsxDEV19("box", {
|
|
6148
|
+
flexDirection: "column",
|
|
6149
|
+
border: true,
|
|
6150
|
+
borderStyle: "rounded",
|
|
6151
|
+
borderColor: theme.border,
|
|
6152
|
+
padding: 1,
|
|
6153
|
+
marginBottom: 1,
|
|
6154
|
+
children: [
|
|
6155
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6156
|
+
fg: theme.accent,
|
|
6157
|
+
bold: true,
|
|
6158
|
+
children: "METRICS EXPLAINED"
|
|
6159
|
+
}, undefined, false, undefined, this),
|
|
6160
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6161
|
+
fg: theme.text,
|
|
6162
|
+
children: " "
|
|
6163
|
+
}, undefined, false, undefined, this),
|
|
6164
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6165
|
+
fg: theme.secondary,
|
|
6166
|
+
bold: true,
|
|
6167
|
+
children: "TPS (Tokens Per Second)"
|
|
6168
|
+
}, undefined, false, undefined, this),
|
|
6169
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6170
|
+
fg: theme.text,
|
|
6171
|
+
children: " How fast a model generates tokens after the first one."
|
|
6172
|
+
}, undefined, false, undefined, this),
|
|
6173
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6174
|
+
fg: theme.dim,
|
|
6175
|
+
children: " Formula: output_tokens / generation_time"
|
|
6176
|
+
}, undefined, false, undefined, this),
|
|
6177
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6178
|
+
fg: theme.success,
|
|
6179
|
+
children: " \u2192 Higher is better (faster streaming)"
|
|
6180
|
+
}, undefined, false, undefined, this),
|
|
6181
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6182
|
+
fg: theme.text,
|
|
6183
|
+
children: " "
|
|
6184
|
+
}, undefined, false, undefined, this),
|
|
6185
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6186
|
+
fg: theme.secondary,
|
|
6187
|
+
bold: true,
|
|
6188
|
+
children: "TTFT (Time To First Token)"
|
|
6189
|
+
}, undefined, false, undefined, this),
|
|
6190
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6191
|
+
fg: theme.text,
|
|
6192
|
+
children: " Time from request to receiving the first token."
|
|
6193
|
+
}, undefined, false, undefined, this),
|
|
6194
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6195
|
+
fg: theme.dim,
|
|
6196
|
+
children: " Formula: first_token_time - request_start_time"
|
|
6197
|
+
}, undefined, false, undefined, this),
|
|
6198
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6199
|
+
fg: theme.success,
|
|
6200
|
+
children: " \u2192 Lower is better (less waiting)"
|
|
6201
|
+
}, undefined, false, undefined, this),
|
|
6202
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6203
|
+
fg: theme.text,
|
|
6204
|
+
children: " "
|
|
6205
|
+
}, undefined, false, undefined, this),
|
|
6206
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6207
|
+
fg: theme.secondary,
|
|
6208
|
+
bold: true,
|
|
6209
|
+
children: "F1000 (First to 1000)"
|
|
6210
|
+
}, undefined, false, undefined, this),
|
|
6211
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6212
|
+
fg: theme.text,
|
|
6213
|
+
children: " Time to complete 1000 agentic requests (~300 tokens each)."
|
|
6214
|
+
}, undefined, false, undefined, this),
|
|
6215
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6216
|
+
fg: theme.dim,
|
|
6217
|
+
children: " Formula: 1000 \xD7 (TTFT + 300/TPS) = hours"
|
|
6218
|
+
}, undefined, false, undefined, this),
|
|
6219
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6220
|
+
fg: theme.success,
|
|
6221
|
+
children: " \u2192 Lower is better"
|
|
6222
|
+
}, undefined, false, undefined, this),
|
|
6223
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6224
|
+
fg: theme.text,
|
|
6225
|
+
children: " "
|
|
6226
|
+
}, undefined, false, undefined, this),
|
|
6227
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6228
|
+
fg: theme.text,
|
|
6229
|
+
children: " Why it matters: In agentic coding (Cursor, Copilot, OpenCode),"
|
|
6230
|
+
}, undefined, false, undefined, this),
|
|
6231
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6232
|
+
fg: theme.text,
|
|
6233
|
+
children: " models make hundreds of tool calls \u2014 TTFT adds up massively."
|
|
6234
|
+
}, undefined, false, undefined, this),
|
|
6235
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6236
|
+
fg: theme.text,
|
|
6237
|
+
children: " A 30 tok/s + 1s TTFT model can match a 60 tok/s + 6s TTFT model."
|
|
6238
|
+
}, undefined, false, undefined, this),
|
|
6239
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6240
|
+
fg: theme.text,
|
|
6241
|
+
children: " "
|
|
6242
|
+
}, undefined, false, undefined, this),
|
|
6243
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6244
|
+
fg: theme.secondary,
|
|
6245
|
+
bold: true,
|
|
6246
|
+
children: "Total Time"
|
|
6247
|
+
}, undefined, false, undefined, this),
|
|
6248
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6249
|
+
fg: theme.text,
|
|
6250
|
+
children: " Complete request duration from start to finish."
|
|
6251
|
+
}, undefined, false, undefined, this),
|
|
6252
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6253
|
+
fg: theme.dim,
|
|
6254
|
+
children: " Includes: connection, TTFT, and generation time"
|
|
6255
|
+
}, undefined, false, undefined, this)
|
|
6256
|
+
]
|
|
6257
|
+
}, undefined, true, undefined, this),
|
|
6258
|
+
/* @__PURE__ */ jsxDEV19("box", {
|
|
6259
|
+
flexDirection: "column",
|
|
6260
|
+
border: true,
|
|
6261
|
+
borderStyle: "rounded",
|
|
6262
|
+
borderColor: theme.border,
|
|
6263
|
+
padding: 1,
|
|
6264
|
+
marginBottom: 1,
|
|
6265
|
+
children: [
|
|
6266
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6267
|
+
fg: theme.accent,
|
|
6268
|
+
bold: true,
|
|
6269
|
+
children: "LINKS"
|
|
6270
|
+
}, undefined, false, undefined, this),
|
|
6271
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6272
|
+
fg: theme.text,
|
|
6273
|
+
children: " "
|
|
6274
|
+
}, undefined, false, undefined, this),
|
|
6275
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6276
|
+
fg: theme.secondary,
|
|
6277
|
+
bold: true,
|
|
6278
|
+
children: "Discord Community"
|
|
6279
|
+
}, undefined, false, undefined, this),
|
|
6280
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6281
|
+
fg: theme.text,
|
|
6282
|
+
children: " https://discord.gg/6S7HwCxbMy"
|
|
6283
|
+
}, undefined, false, undefined, this),
|
|
6284
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6285
|
+
fg: theme.dim,
|
|
6286
|
+
children: " Join for help, updates, and discussions"
|
|
6287
|
+
}, undefined, false, undefined, this),
|
|
6288
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6289
|
+
fg: theme.text,
|
|
6290
|
+
children: " "
|
|
6291
|
+
}, undefined, false, undefined, this),
|
|
6292
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6293
|
+
fg: theme.secondary,
|
|
6294
|
+
bold: true,
|
|
6295
|
+
children: "Website & Leaderboard"
|
|
6296
|
+
}, undefined, false, undefined, this),
|
|
6297
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6298
|
+
fg: theme.text,
|
|
6299
|
+
children: " https://ai-speedometer.oliveowl.xyz/"
|
|
6300
|
+
}, undefined, false, undefined, this),
|
|
6301
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6302
|
+
fg: theme.dim,
|
|
6303
|
+
children: " Track OSS model speeds over time"
|
|
6304
|
+
}, undefined, false, undefined, this),
|
|
6305
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6306
|
+
fg: theme.text,
|
|
6307
|
+
children: " "
|
|
6308
|
+
}, undefined, false, undefined, this),
|
|
6309
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6310
|
+
fg: theme.secondary,
|
|
6311
|
+
bold: true,
|
|
6312
|
+
children: "GitHub"
|
|
6313
|
+
}, undefined, false, undefined, this),
|
|
6314
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6315
|
+
fg: theme.text,
|
|
6316
|
+
children: " https://github.com/anomaly/ai-speedometer"
|
|
6317
|
+
}, undefined, false, undefined, this),
|
|
6318
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6319
|
+
fg: theme.dim,
|
|
6320
|
+
children: " Source code, issues, contributions"
|
|
6321
|
+
}, undefined, false, undefined, this)
|
|
6322
|
+
]
|
|
6323
|
+
}, undefined, true, undefined, this),
|
|
6324
|
+
/* @__PURE__ */ jsxDEV19("box", {
|
|
6325
|
+
flexDirection: "column",
|
|
6326
|
+
border: true,
|
|
6327
|
+
borderStyle: "rounded",
|
|
6328
|
+
borderColor: theme.border,
|
|
6329
|
+
padding: 1,
|
|
6330
|
+
marginBottom: 1,
|
|
6331
|
+
children: [
|
|
6332
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6333
|
+
fg: theme.accent,
|
|
6334
|
+
bold: true,
|
|
6335
|
+
children: "TIPS"
|
|
6336
|
+
}, undefined, false, undefined, this),
|
|
6337
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6338
|
+
fg: theme.text,
|
|
6339
|
+
children: " "
|
|
6340
|
+
}, undefined, false, undefined, this),
|
|
6341
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6342
|
+
fg: theme.text,
|
|
6343
|
+
children: " \u2022 Press [T] anytime to open the theme picker"
|
|
6344
|
+
}, undefined, false, undefined, this),
|
|
6345
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6346
|
+
fg: theme.text,
|
|
6347
|
+
children: " \u2022 Use --log flag to save raw SSE data for debugging"
|
|
6348
|
+
}, undefined, false, undefined, this),
|
|
6349
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6350
|
+
fg: theme.text,
|
|
6351
|
+
children: " \u2022 Run headless with ai-speedometer-headless for CI/CD"
|
|
6352
|
+
}, undefined, false, undefined, this),
|
|
6353
|
+
/* @__PURE__ */ jsxDEV19("text", {
|
|
6354
|
+
fg: theme.text,
|
|
6355
|
+
children: " \u2022 [*] in results means token count was estimated"
|
|
6356
|
+
}, undefined, false, undefined, this)
|
|
6357
|
+
]
|
|
6358
|
+
}, undefined, true, undefined, this),
|
|
6359
|
+
/* @__PURE__ */ jsxDEV19("box", {
|
|
6360
|
+
flexDirection: "row",
|
|
6361
|
+
justifyContent: "center",
|
|
6362
|
+
marginTop: 1,
|
|
6363
|
+
children: /* @__PURE__ */ jsxDEV19("text", {
|
|
6364
|
+
fg: theme.dim,
|
|
6365
|
+
children: "Press [Q] or [Esc] to return to main menu"
|
|
6366
|
+
}, undefined, false, undefined, this)
|
|
6367
|
+
}, undefined, false, undefined, this)
|
|
6368
|
+
]
|
|
6369
|
+
}, undefined, true, undefined, this)
|
|
6370
|
+
}, undefined, false, undefined, this)
|
|
6371
|
+
]
|
|
6372
|
+
}, undefined, true, undefined, this)
|
|
6373
|
+
}, undefined, false, undefined, this);
|
|
6374
|
+
}
|
|
6375
|
+
var init_FAQScreen = __esm(() => {
|
|
6376
|
+
init_useAppKeyboard();
|
|
6377
|
+
init_AppContext();
|
|
6378
|
+
init_ThemeContext();
|
|
6379
|
+
});
|
|
6380
|
+
|
|
5936
6381
|
// src/tui/App.tsx
|
|
5937
6382
|
import { useKeyboard as useKeyboard3, useRenderer as useRenderer3 } from "@opentui/react";
|
|
5938
|
-
import { jsxDEV as
|
|
6383
|
+
import { jsxDEV as jsxDEV20 } from "@opentui/react/jsx-dev-runtime";
|
|
5939
6384
|
function getHints(screen, benchResults) {
|
|
5940
6385
|
switch (screen) {
|
|
5941
6386
|
case "main-menu":
|
|
@@ -5950,6 +6395,8 @@ function getHints(screen, benchResults) {
|
|
|
5950
6395
|
}
|
|
5951
6396
|
case "list-providers":
|
|
5952
6397
|
return ["[\u2191\u2193] scroll", "[Q] back"];
|
|
6398
|
+
case "faq":
|
|
6399
|
+
return ["[\u2191\u2193] scroll", "[Q] back"];
|
|
5953
6400
|
case "add-verified":
|
|
5954
6401
|
return ["[\u2191\u2193] navigate", "[Enter] select", "[Q] back"];
|
|
5955
6402
|
case "add-custom":
|
|
@@ -5964,21 +6411,23 @@ function ActiveScreen() {
|
|
|
5964
6411
|
const { state } = useAppContext();
|
|
5965
6412
|
switch (state.screen) {
|
|
5966
6413
|
case "main-menu":
|
|
5967
|
-
return /* @__PURE__ */
|
|
6414
|
+
return /* @__PURE__ */ jsxDEV20(MainMenuScreen, {}, undefined, false, undefined, this);
|
|
5968
6415
|
case "model-menu":
|
|
5969
|
-
return /* @__PURE__ */
|
|
6416
|
+
return /* @__PURE__ */ jsxDEV20(ModelMenuScreen, {}, undefined, false, undefined, this);
|
|
5970
6417
|
case "model-select":
|
|
5971
|
-
return /* @__PURE__ */
|
|
6418
|
+
return /* @__PURE__ */ jsxDEV20(ModelSelectScreen, {}, undefined, false, undefined, this);
|
|
5972
6419
|
case "benchmark":
|
|
5973
|
-
return /* @__PURE__ */
|
|
6420
|
+
return /* @__PURE__ */ jsxDEV20(BenchmarkScreen, {}, undefined, false, undefined, this);
|
|
5974
6421
|
case "add-verified":
|
|
5975
|
-
return /* @__PURE__ */
|
|
6422
|
+
return /* @__PURE__ */ jsxDEV20(AddVerifiedScreen, {}, undefined, false, undefined, this);
|
|
5976
6423
|
case "add-custom":
|
|
5977
|
-
return /* @__PURE__ */
|
|
6424
|
+
return /* @__PURE__ */ jsxDEV20(AddCustomScreen, {}, undefined, false, undefined, this);
|
|
5978
6425
|
case "add-models":
|
|
5979
|
-
return /* @__PURE__ */
|
|
6426
|
+
return /* @__PURE__ */ jsxDEV20(AddModelsScreen, {}, undefined, false, undefined, this);
|
|
5980
6427
|
case "list-providers":
|
|
5981
|
-
return /* @__PURE__ */
|
|
6428
|
+
return /* @__PURE__ */ jsxDEV20(ListProvidersScreen, {}, undefined, false, undefined, this);
|
|
6429
|
+
case "faq":
|
|
6430
|
+
return /* @__PURE__ */ jsxDEV20(FAQScreen, {}, undefined, false, undefined, this);
|
|
5982
6431
|
}
|
|
5983
6432
|
}
|
|
5984
6433
|
function Shell() {
|
|
@@ -5995,36 +6444,36 @@ function Shell() {
|
|
|
5995
6444
|
setModalOpen(!modalOpen);
|
|
5996
6445
|
}
|
|
5997
6446
|
});
|
|
5998
|
-
return /* @__PURE__ */
|
|
6447
|
+
return /* @__PURE__ */ jsxDEV20("box", {
|
|
5999
6448
|
flexDirection: "column",
|
|
6000
6449
|
height: "100%",
|
|
6001
6450
|
width: "100%",
|
|
6002
6451
|
backgroundColor: theme.background,
|
|
6003
6452
|
children: [
|
|
6004
|
-
/* @__PURE__ */
|
|
6453
|
+
/* @__PURE__ */ jsxDEV20(Header, {
|
|
6005
6454
|
screen: state.screen
|
|
6006
6455
|
}, undefined, false, undefined, this),
|
|
6007
|
-
/* @__PURE__ */
|
|
6456
|
+
/* @__PURE__ */ jsxDEV20("box", {
|
|
6008
6457
|
flexGrow: 1,
|
|
6009
6458
|
flexDirection: "column",
|
|
6010
|
-
children: /* @__PURE__ */
|
|
6459
|
+
children: /* @__PURE__ */ jsxDEV20(ActiveScreen, {}, undefined, false, undefined, this)
|
|
6011
6460
|
}, undefined, false, undefined, this),
|
|
6012
|
-
/* @__PURE__ */
|
|
6461
|
+
/* @__PURE__ */ jsxDEV20(Footer, {
|
|
6013
6462
|
hints: getHints(state.screen, state.benchResults)
|
|
6014
6463
|
}, undefined, false, undefined, this),
|
|
6015
|
-
modalOpen && /* @__PURE__ */
|
|
6464
|
+
modalOpen && /* @__PURE__ */ jsxDEV20(ThemePicker, {
|
|
6016
6465
|
onClose: () => setModalOpen(false)
|
|
6017
6466
|
}, undefined, false, undefined, this)
|
|
6018
6467
|
]
|
|
6019
6468
|
}, undefined, true, undefined, this);
|
|
6020
6469
|
}
|
|
6021
6470
|
function App({ logMode = false, theme = "tokyonight" }) {
|
|
6022
|
-
return /* @__PURE__ */
|
|
6471
|
+
return /* @__PURE__ */ jsxDEV20(ThemeProvider, {
|
|
6023
6472
|
name: theme,
|
|
6024
|
-
children: /* @__PURE__ */
|
|
6025
|
-
children: /* @__PURE__ */
|
|
6473
|
+
children: /* @__PURE__ */ jsxDEV20(ModalProvider, {
|
|
6474
|
+
children: /* @__PURE__ */ jsxDEV20(AppProvider, {
|
|
6026
6475
|
logMode,
|
|
6027
|
-
children: /* @__PURE__ */
|
|
6476
|
+
children: /* @__PURE__ */ jsxDEV20(Shell, {}, undefined, false, undefined, this)
|
|
6028
6477
|
}, undefined, false, undefined, this)
|
|
6029
6478
|
}, undefined, false, undefined, this)
|
|
6030
6479
|
}, undefined, false, undefined, this);
|
|
@@ -6044,6 +6493,7 @@ var init_App = __esm(() => {
|
|
|
6044
6493
|
init_AddCustomScreen();
|
|
6045
6494
|
init_AddModelsScreen();
|
|
6046
6495
|
init_ListProvidersScreen();
|
|
6496
|
+
init_FAQScreen();
|
|
6047
6497
|
});
|
|
6048
6498
|
|
|
6049
6499
|
// src/tui/index.tsx
|
|
@@ -6053,7 +6503,7 @@ __export(exports_tui, {
|
|
|
6053
6503
|
});
|
|
6054
6504
|
import { createCliRenderer } from "@opentui/core";
|
|
6055
6505
|
import { createRoot } from "@opentui/react";
|
|
6056
|
-
import { jsxDEV as
|
|
6506
|
+
import { jsxDEV as jsxDEV21 } from "@opentui/react/jsx-dev-runtime";
|
|
6057
6507
|
async function startTui(logMode = false) {
|
|
6058
6508
|
const { readThemeFromConfig: readThemeFromConfig2 } = await Promise.resolve().then(() => (init_ai_config(), exports_ai_config));
|
|
6059
6509
|
const theme = await readThemeFromConfig2();
|
|
@@ -6070,7 +6520,7 @@ async function startTui(logMode = false) {
|
|
|
6070
6520
|
renderer.destroy();
|
|
6071
6521
|
process.exit(0);
|
|
6072
6522
|
});
|
|
6073
|
-
createRoot(renderer).render(/* @__PURE__ */
|
|
6523
|
+
createRoot(renderer).render(/* @__PURE__ */ jsxDEV21(App, {
|
|
6074
6524
|
logMode,
|
|
6075
6525
|
theme
|
|
6076
6526
|
}, undefined, false, undefined, this));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-speedometer",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
|
|
5
5
|
"bin": {
|
|
6
6
|
"ai-speedometer": "dist/ai-speedometer",
|