ai-speedometer 2.2.1 → 2.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/ai-speedometer +504 -45
  2. package/package.json +1 -1
@@ -1932,6 +1932,7 @@ async function benchmarkSingleModelRest(model, logger) {
1932
1932
  const finalInputTokens = inputTokens || Math.round(TEST_PROMPT.length / 4);
1933
1933
  const totalTokens = finalInputTokens + finalOutputTokens;
1934
1934
  const tokensPerSecond = generationTime > 0 ? finalOutputTokens / generationTime * 1000 : 0;
1935
+ const f1000 = tokensPerSecond > 0 ? 1000 * (timeToFirstToken / 1000 + 300 / tokensPerSecond) / 3600 : Infinity;
1935
1936
  return {
1936
1937
  model: model.name,
1937
1938
  provider: model.providerName,
@@ -1939,6 +1940,7 @@ async function benchmarkSingleModelRest(model, logger) {
1939
1940
  timeToFirstToken,
1940
1941
  tokenCount: finalOutputTokens,
1941
1942
  tokensPerSecond,
1943
+ f1000,
1942
1944
  promptTokens: finalInputTokens,
1943
1945
  totalTokens,
1944
1946
  usedEstimateForOutput,
@@ -1954,6 +1956,7 @@ async function benchmarkSingleModelRest(model, logger) {
1954
1956
  timeToFirstToken: 0,
1955
1957
  tokenCount: 0,
1956
1958
  tokensPerSecond: 0,
1959
+ f1000: Infinity,
1957
1960
  promptTokens: 0,
1958
1961
  totalTokens: 0,
1959
1962
  usedEstimateForOutput: true,
@@ -2019,6 +2022,8 @@ function buildJsonOutput(providerName, providerId, modelName, modelId, result, f
2019
2022
  timeToFirstToken: result.timeToFirstToken,
2020
2023
  timeToFirstTokenSeconds: result.timeToFirstToken / 1000,
2021
2024
  tokensPerSecond: result.tokensPerSecond,
2025
+ f1000: result.f1000,
2026
+ f1000Hours: result.f1000 === Infinity ? null : result.f1000,
2022
2027
  outputTokens: result.tokenCount,
2023
2028
  promptTokens: result.promptTokens,
2024
2029
  totalTokens: result.totalTokens,
@@ -3011,7 +3016,7 @@ var package_default;
3011
3016
  var init_package = __esm(() => {
3012
3017
  package_default = {
3013
3018
  name: "ai-speedometer",
3014
- version: "2.2.1",
3019
+ version: "2.3.1",
3015
3020
  description: "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
3016
3021
  bin: {
3017
3022
  "ai-speedometer": "dist/ai-speedometer",
@@ -3155,6 +3160,7 @@ function MainMenuScreen() {
3155
3160
  const ITEMS = [
3156
3161
  { label: "\u26A1 Run Benchmark", desc: "test model speed & throughput", color: theme.accent },
3157
3162
  { label: "\u2699 Manage Models", desc: "add providers and configure", color: theme.secondary },
3163
+ { label: "? FAQ / Learn", desc: "how metrics work & resources", color: theme.primary },
3158
3164
  { label: "\u2715 Exit", desc: "quit the application", color: theme.error }
3159
3165
  ];
3160
3166
  useAppKeyboard((key) => {
@@ -3168,6 +3174,8 @@ function MainMenuScreen() {
3168
3174
  else if (cursor === 1)
3169
3175
  navigate("model-menu");
3170
3176
  else if (cursor === 2)
3177
+ navigate("faq");
3178
+ else if (cursor === 3)
3171
3179
  renderer.destroy();
3172
3180
  }
3173
3181
  });
@@ -3202,7 +3210,7 @@ function MainMenuScreen() {
3202
3210
  borderStyle: "rounded",
3203
3211
  borderColor: theme.border,
3204
3212
  backgroundColor: theme.background,
3205
- width: 46,
3213
+ width: 48,
3206
3214
  children: ITEMS.map((item, i) => {
3207
3215
  const active = i === cursor;
3208
3216
  return /* @__PURE__ */ jsxDEV7("box", {
@@ -3712,8 +3720,9 @@ var init_ModelSelectScreen = __esm(() => {
3712
3720
 
3713
3721
  // src/tui/components/BarChart.tsx
3714
3722
  import { jsxDEV as jsxDEV11 } from "@opentui/react/jsx-dev-runtime";
3715
- function BarChart({ value, max, width, color }) {
3716
- const filled = max === 0 ? 0 : Math.round(value / max * width);
3723
+ function BarChart({ value, max, width, color, inverted = false }) {
3724
+ const normalizedValue = inverted && max > 0 ? Math.max(0, max - value) : value;
3725
+ const filled = max === 0 ? 0 : Math.round(normalizedValue / max * width);
3717
3726
  const empty = width - filled;
3718
3727
  const bar = "\u2588".repeat(filled) + "\u2591".repeat(empty);
3719
3728
  return /* @__PURE__ */ jsxDEV11("text", {
@@ -3817,13 +3826,17 @@ function trunc(s, w) {
3817
3826
  function ResultsTable({ results, pendingCount }) {
3818
3827
  const theme = useTheme();
3819
3828
  const sorted = [...results].sort((a, b) => b.tokensPerSecond - a.tokensPerSecond);
3820
- const C = { rank: 4, model: 18, prov: 12, time: 10, ttft: 8, tps: 11, out: 8, inp: 8, tot: 8 };
3821
- const totalW = C.rank + C.model + C.prov + C.time + C.ttft + C.tps + C.out + C.inp + C.tot + 9;
3829
+ const C = { rank: 4, model: 16, prov: 10, time: 8, ttft: 7, tps: 9, f1000: 8, out: 6, inp: 6, tot: 6 };
3830
+ const totalW = C.rank + C.model + C.prov + C.time + C.ttft + C.tps + C.f1000 + C.out + C.inp + C.tot + 10;
3822
3831
  const sep = "\u2500".repeat(totalW);
3823
- function row(rank, model, prov, time, ttft, tps, out, inp, tot) {
3824
- return lpad(rank, C.rank) + " \u2502 " + rpad(model, C.model) + " \u2502 " + rpad(prov, C.prov) + " \u2502 " + lpad(time, C.time) + " \u2502 " + lpad(ttft, C.ttft) + " \u2502 " + lpad(tps, C.tps) + " \u2502 " + lpad(out, C.out) + " \u2502 " + lpad(inp, C.inp) + " \u2502 " + lpad(tot, C.tot);
3832
+ const maxTps = Math.max(...results.map((r) => r.tokensPerSecond), 0);
3833
+ const minTtft = Math.min(...results.map((r) => r.timeToFirstToken), Infinity);
3834
+ const validF1000s = results.map((r) => r.f1000).filter((f) => f !== Infinity);
3835
+ const minF1000 = validF1000s.length > 0 ? Math.min(...validF1000s) : Infinity;
3836
+ function row(rank, model, prov, time, ttft, tps, f1000, out, inp, tot) {
3837
+ return lpad(rank, C.rank) + " \u2502 " + rpad(model, C.model) + " \u2502 " + rpad(prov, C.prov) + " \u2502 " + lpad(time, C.time) + " \u2502 " + lpad(ttft, C.ttft) + " \u2502 " + lpad(tps, C.tps) + " \u2502 " + lpad(f1000, C.f1000) + " \u2502 " + lpad(out, C.out) + " \u2502 " + lpad(inp, C.inp) + " \u2502 " + lpad(tot, C.tot);
3825
3838
  }
3826
- const header = row("#", "Model", "Provider", "Time(s)", "TTFT(s)", "Tokens/Sec", "Out", "In", "Total");
3839
+ const header = row("#", "Model", "Provider", "Time(s)", "TTFT(s)", "Tok/s", "F1000(h)", "Out", "In", "Total");
3827
3840
  return /* @__PURE__ */ jsxDEV13("box", {
3828
3841
  flexDirection: "column",
3829
3842
  paddingLeft: 1,
@@ -3845,21 +3858,87 @@ function ResultsTable({ results, pendingCount }) {
3845
3858
  }, undefined, false, undefined, this),
3846
3859
  sorted.map((r, i) => {
3847
3860
  const rank = `${i + 1}`;
3848
- const timeSec = (r.totalTime / 1000).toFixed(2);
3861
+ const timeSec = (r.totalTime / 1000).toFixed(1);
3849
3862
  const ttftSec = (r.timeToFirstToken / 1000).toFixed(2);
3850
- const tps = r.tokensPerSecond.toFixed(1);
3851
- const outTok = r.tokenCount.toString() + (r.usedEstimateForOutput ? "[e]" : "");
3852
- const inTok = r.promptTokens.toString() + (r.usedEstimateForInput ? "[e]" : "");
3853
- const totTok = r.totalTokens.toString() + (r.usedEstimateForOutput || r.usedEstimateForInput ? "[e]" : "");
3863
+ const tps = r.tokensPerSecond.toFixed(0);
3864
+ const f1000Val = r.f1000 === Infinity ? "\u221E" : r.f1000.toFixed(2);
3865
+ const outTok = r.tokenCount.toString() + (r.usedEstimateForOutput ? "*" : "");
3866
+ const inTok = r.promptTokens.toString() + (r.usedEstimateForInput ? "*" : "");
3867
+ const totTok = r.totalTokens.toString() + (r.usedEstimateForOutput || r.usedEstimateForInput ? "*" : "");
3854
3868
  const hasEst = r.usedEstimateForOutput || r.usedEstimateForInput;
3855
- const line = row(rank, trunc(r.model, C.model), trunc(r.provider, C.prov), timeSec, ttftSec, tps, outTok, inTok, totTok);
3869
+ const isBestTps = r.tokensPerSecond === maxTps && maxTps > 0;
3870
+ const isBestTtft = r.timeToFirstToken === minTtft;
3871
+ const isBestF1000 = r.f1000 === minF1000 && r.f1000 !== Infinity;
3856
3872
  return /* @__PURE__ */ jsxDEV13("box", {
3857
3873
  height: 1,
3858
3874
  flexDirection: "row",
3859
3875
  children: [
3876
+ /* @__PURE__ */ jsxDEV13("text", {
3877
+ fg: theme.dim,
3878
+ children: [
3879
+ lpad(rank, C.rank),
3880
+ " \u2502 "
3881
+ ]
3882
+ }, undefined, true, undefined, this),
3860
3883
  /* @__PURE__ */ jsxDEV13("text", {
3861
3884
  fg: theme.text,
3862
- children: line
3885
+ children: [
3886
+ rpad(trunc(r.model, C.model), C.model),
3887
+ " \u2502 "
3888
+ ]
3889
+ }, undefined, true, undefined, this),
3890
+ /* @__PURE__ */ jsxDEV13("text", {
3891
+ fg: theme.dim,
3892
+ children: [
3893
+ rpad(trunc(r.provider, C.prov), C.prov),
3894
+ " \u2502 "
3895
+ ]
3896
+ }, undefined, true, undefined, this),
3897
+ /* @__PURE__ */ jsxDEV13("text", {
3898
+ fg: theme.dim,
3899
+ children: [
3900
+ lpad(timeSec, C.time),
3901
+ " \u2502 "
3902
+ ]
3903
+ }, undefined, true, undefined, this),
3904
+ /* @__PURE__ */ jsxDEV13("text", {
3905
+ fg: isBestTtft ? theme.success : theme.dim,
3906
+ children: [
3907
+ lpad(ttftSec, C.ttft),
3908
+ " \u2502 "
3909
+ ]
3910
+ }, undefined, true, undefined, this),
3911
+ /* @__PURE__ */ jsxDEV13("text", {
3912
+ fg: isBestTps ? theme.success : theme.dim,
3913
+ children: [
3914
+ lpad(tps, C.tps),
3915
+ " \u2502 "
3916
+ ]
3917
+ }, undefined, true, undefined, this),
3918
+ /* @__PURE__ */ jsxDEV13("text", {
3919
+ fg: isBestF1000 ? theme.success : theme.dim,
3920
+ children: [
3921
+ lpad(f1000Val, C.f1000),
3922
+ " \u2502 "
3923
+ ]
3924
+ }, undefined, true, undefined, this),
3925
+ /* @__PURE__ */ jsxDEV13("text", {
3926
+ fg: theme.dim,
3927
+ children: [
3928
+ lpad(outTok, C.out),
3929
+ " \u2502 "
3930
+ ]
3931
+ }, undefined, true, undefined, this),
3932
+ /* @__PURE__ */ jsxDEV13("text", {
3933
+ fg: theme.dim,
3934
+ children: [
3935
+ lpad(inTok, C.inp),
3936
+ " \u2502 "
3937
+ ]
3938
+ }, undefined, true, undefined, this),
3939
+ /* @__PURE__ */ jsxDEV13("text", {
3940
+ fg: theme.dim,
3941
+ children: lpad(totTok, C.tot)
3863
3942
  }, undefined, false, undefined, this),
3864
3943
  hasEst && /* @__PURE__ */ jsxDEV13("text", {
3865
3944
  fg: theme.warning,
@@ -3914,16 +3993,14 @@ function BenchmarkScreen() {
3914
3993
  const [modelStates, setModelStates] = useState8([]);
3915
3994
  const [spinnerFrame, setSpinnerFrame] = useState8(0);
3916
3995
  const [allDone, setAllDone] = useState8(false);
3996
+ const [runKey, setRunKey] = useState8(0);
3917
3997
  const spinnerRef = useRef4(null);
3918
- const startedRef = useRef4(false);
3919
3998
  useEffect6(() => {
3920
- if (startedRef.current)
3921
- return;
3922
- startedRef.current = true;
3923
3999
  const models = state.benchResults.map((r) => r.model);
3924
4000
  if (models.length === 0)
3925
4001
  return;
3926
4002
  setModelStates(models.map((m) => ({ model: m, status: "pending" })));
4003
+ setAllDone(false);
3927
4004
  spinnerRef.current = setInterval(() => setSpinnerFrame((f) => f + 1), 80);
3928
4005
  setModelStates((prev) => prev.map((s) => ({ ...s, status: "running", startedAt: Date.now() })));
3929
4006
  async function runAll() {
@@ -3966,7 +4043,14 @@ function BenchmarkScreen() {
3966
4043
  spinnerRef.current = null;
3967
4044
  }
3968
4045
  };
3969
- }, []);
4046
+ }, [runKey]);
4047
+ const rerun = () => {
4048
+ if (spinnerRef.current) {
4049
+ clearInterval(spinnerRef.current);
4050
+ spinnerRef.current = null;
4051
+ }
4052
+ setRunKey((k) => k + 1);
4053
+ };
3970
4054
  const done = modelStates.filter((m) => m.status === "done");
3971
4055
  const running = modelStates.filter((m) => m.status === "running");
3972
4056
  const pending = modelStates.filter((m) => m.status === "pending");
@@ -3975,7 +4059,9 @@ function BenchmarkScreen() {
3975
4059
  const maxTtft = Math.max(...done.map((m) => (m.result?.timeToFirstToken ?? 0) / 1000), 1);
3976
4060
  const tpsRanked = done.slice().sort((a, b) => (b.result?.tokensPerSecond ?? 0) - (a.result?.tokensPerSecond ?? 0));
3977
4061
  const ttftRanked = done.slice().sort((a, b) => (a.result?.timeToFirstToken ?? 0) - (b.result?.timeToFirstToken ?? 0));
4062
+ const f1000Ranked = done.slice().sort((a, b) => (a.result?.f1000 ?? Infinity) - (b.result?.f1000 ?? Infinity));
3978
4063
  const maxTtftForBar = Math.max(...done.map((m) => (m.result?.timeToFirstToken ?? 0) / 1000), 1);
4064
+ const maxF1000 = Math.max(...done.map((m) => m.result?.f1000 ?? 0), 1);
3979
4065
  const doneResults = tpsRanked.map((m) => m.result);
3980
4066
  const pendingCount = running.length + pending.length;
3981
4067
  const allRows = useMemo(() => {
@@ -4217,6 +4303,103 @@ function BenchmarkScreen() {
4217
4303
  }, `ttft-${s.model.id}-${s.model.providerId}`, true, undefined, this));
4218
4304
  }
4219
4305
  }
4306
+ if (f1000Ranked.length > 0) {
4307
+ rows.push(/* @__PURE__ */ jsxDEV14("box", {
4308
+ height: 1,
4309
+ backgroundColor: theme.border
4310
+ }, "div-f1000", false, undefined, this));
4311
+ rows.push(/* @__PURE__ */ jsxDEV14("box", {
4312
+ height: 1,
4313
+ flexDirection: "row",
4314
+ paddingLeft: 1,
4315
+ children: /* @__PURE__ */ jsxDEV14("text", {
4316
+ fg: theme.primary,
4317
+ children: " F1000 RANKING - First to 1000 Requests (lower is better) "
4318
+ }, undefined, false, undefined, this)
4319
+ }, "hdr-f1000", false, undefined, this));
4320
+ for (const [i, s] of f1000Ranked.entries()) {
4321
+ const rank = i + 1;
4322
+ const rankFg = rank === 1 ? theme.accent : rank === 2 ? theme.secondary : theme.dim;
4323
+ const f1000 = s.result?.f1000 ?? Infinity;
4324
+ const f1000Str = f1000 === Infinity ? "\u221E" : f1000.toFixed(2);
4325
+ const ttft = (s.result?.timeToFirstToken ?? 0) / 1000;
4326
+ const tps = s.result?.tokensPerSecond ?? 0;
4327
+ const badge = rankBadge(rank).padStart(3);
4328
+ const modelCol = s.model.name.padEnd(18).slice(0, 18);
4329
+ const provCol = s.model.providerName.padEnd(12).slice(0, 12);
4330
+ rows.push(/* @__PURE__ */ jsxDEV14("box", {
4331
+ height: 1,
4332
+ flexDirection: "row",
4333
+ paddingLeft: 2,
4334
+ children: [
4335
+ /* @__PURE__ */ jsxDEV14("text", {
4336
+ fg: rankFg,
4337
+ children: [
4338
+ badge,
4339
+ " "
4340
+ ]
4341
+ }, undefined, true, undefined, this),
4342
+ /* @__PURE__ */ jsxDEV14("text", {
4343
+ fg: theme.dim,
4344
+ children: " \u2502 "
4345
+ }, undefined, false, undefined, this),
4346
+ /* @__PURE__ */ jsxDEV14("text", {
4347
+ fg: theme.primary,
4348
+ children: [
4349
+ f1000Str.padStart(7),
4350
+ "h "
4351
+ ]
4352
+ }, undefined, true, undefined, this),
4353
+ /* @__PURE__ */ jsxDEV14("text", {
4354
+ fg: theme.dim,
4355
+ children: " \u2502 "
4356
+ }, undefined, false, undefined, this),
4357
+ /* @__PURE__ */ jsxDEV14("text", {
4358
+ fg: theme.secondary,
4359
+ children: [
4360
+ ttft.toFixed(2).padStart(5),
4361
+ "s "
4362
+ ]
4363
+ }, undefined, true, undefined, this),
4364
+ /* @__PURE__ */ jsxDEV14("text", {
4365
+ fg: theme.dim,
4366
+ children: " \u2502 "
4367
+ }, undefined, false, undefined, this),
4368
+ /* @__PURE__ */ jsxDEV14("text", {
4369
+ fg: theme.accent,
4370
+ children: [
4371
+ tps.toFixed(0).padStart(5),
4372
+ " tok/s "
4373
+ ]
4374
+ }, undefined, true, undefined, this),
4375
+ /* @__PURE__ */ jsxDEV14("text", {
4376
+ fg: theme.dim,
4377
+ children: " \u2502 "
4378
+ }, undefined, false, undefined, this),
4379
+ /* @__PURE__ */ jsxDEV14("text", {
4380
+ fg: theme.text,
4381
+ children: [
4382
+ modelCol,
4383
+ " "
4384
+ ]
4385
+ }, undefined, true, undefined, this),
4386
+ /* @__PURE__ */ jsxDEV14("text", {
4387
+ fg: theme.dim,
4388
+ children: [
4389
+ provCol,
4390
+ " \u2502 "
4391
+ ]
4392
+ }, undefined, true, undefined, this),
4393
+ /* @__PURE__ */ jsxDEV14(BarChart, {
4394
+ value: f1000 === Infinity ? maxF1000 : f1000,
4395
+ max: maxF1000,
4396
+ width: BAR_W2,
4397
+ color: theme.primary
4398
+ }, undefined, false, undefined, this)
4399
+ ]
4400
+ }, `f1000-${s.model.id}-${s.model.providerId}`, true, undefined, this));
4401
+ }
4402
+ }
4220
4403
  if (allDone && errors.length > 0) {
4221
4404
  rows.push(/* @__PURE__ */ jsxDEV14("box", {
4222
4405
  height: 1,
@@ -4311,10 +4494,14 @@ function BenchmarkScreen() {
4311
4494
  }, "results-empty", false, undefined, this));
4312
4495
  }
4313
4496
  return rows;
4314
- }, [modelStates, allDone, tpsRanked, ttftRanked, doneResults, pendingCount, maxTps, maxTtftForBar, theme]);
4497
+ }, [modelStates, allDone, tpsRanked, ttftRanked, f1000Ranked, doneResults, pendingCount, maxTps, maxTtftForBar, maxF1000, theme]);
4315
4498
  useAppKeyboard((key) => {
4316
4499
  if (!allDone)
4317
4500
  return;
4501
+ if (key.shift && key.name === "r") {
4502
+ rerun();
4503
+ return;
4504
+ }
4318
4505
  if (key.name === "q" || key.name === "return" || key.name === "enter") {
4319
4506
  dispatch({ type: "BENCH_RESET" });
4320
4507
  navigate("main-menu");
@@ -4325,7 +4512,7 @@ function BenchmarkScreen() {
4325
4512
  children: [
4326
4513
  /* @__PURE__ */ jsxDEV14("text", {
4327
4514
  fg: theme.success,
4328
- children: "All done! [Enter]/[Q] return [\u2191\u2193/PgUp/PgDn/wheel] scroll"
4515
+ children: "All done! [R] rerun [Enter]/[Q] return [\u2191\u2193/PgUp/PgDn/wheel] scroll"
4329
4516
  }, undefined, false, undefined, this),
4330
4517
  state.logMode && state.logPath && /* @__PURE__ */ jsxDEV14("text", {
4331
4518
  fg: theme.dim,
@@ -5933,9 +6120,276 @@ var init_ListProvidersScreen = __esm(() => {
5933
6120
  init_ThemeContext();
5934
6121
  });
5935
6122
 
6123
+ // src/tui/screens/FAQScreen.tsx
6124
+ import { jsxDEV as jsxDEV19 } from "@opentui/react/jsx-dev-runtime";
6125
+ function FAQScreen() {
6126
+ const navigate = useNavigate();
6127
+ const theme = useTheme();
6128
+ useAppKeyboard((key) => {
6129
+ if (key.name === "escape" || key.name === "q") {
6130
+ navigate("main-menu");
6131
+ }
6132
+ });
6133
+ return /* @__PURE__ */ jsxDEV19("box", {
6134
+ flexDirection: "column",
6135
+ flexGrow: 1,
6136
+ alignItems: "center",
6137
+ padding: 1,
6138
+ children: /* @__PURE__ */ jsxDEV19("box", {
6139
+ flexDirection: "column",
6140
+ width: 70,
6141
+ children: [
6142
+ /* @__PURE__ */ jsxDEV19("box", {
6143
+ marginBottom: 1,
6144
+ children: /* @__PURE__ */ jsxDEV19("text", {
6145
+ fg: theme.primary,
6146
+ bold: true,
6147
+ children: "FAQ / Learn"
6148
+ }, undefined, false, undefined, this)
6149
+ }, undefined, false, undefined, this),
6150
+ /* @__PURE__ */ jsxDEV19("scrollbox", {
6151
+ focused: true,
6152
+ flexGrow: 1,
6153
+ children: /* @__PURE__ */ jsxDEV19("box", {
6154
+ flexDirection: "column",
6155
+ children: [
6156
+ /* @__PURE__ */ jsxDEV19("box", {
6157
+ flexDirection: "column",
6158
+ border: true,
6159
+ borderStyle: "rounded",
6160
+ borderColor: theme.border,
6161
+ padding: 1,
6162
+ marginBottom: 1,
6163
+ children: [
6164
+ /* @__PURE__ */ jsxDEV19("text", {
6165
+ fg: theme.accent,
6166
+ bold: true,
6167
+ children: "METRICS EXPLAINED"
6168
+ }, undefined, false, undefined, this),
6169
+ /* @__PURE__ */ jsxDEV19("text", {
6170
+ fg: theme.text,
6171
+ children: " "
6172
+ }, undefined, false, undefined, this),
6173
+ /* @__PURE__ */ jsxDEV19("text", {
6174
+ fg: theme.secondary,
6175
+ bold: true,
6176
+ children: "TPS (Tokens Per Second)"
6177
+ }, undefined, false, undefined, this),
6178
+ /* @__PURE__ */ jsxDEV19("text", {
6179
+ fg: theme.text,
6180
+ children: " How fast a model generates tokens after the first one."
6181
+ }, undefined, false, undefined, this),
6182
+ /* @__PURE__ */ jsxDEV19("text", {
6183
+ fg: theme.dim,
6184
+ children: " Formula: output_tokens / generation_time"
6185
+ }, undefined, false, undefined, this),
6186
+ /* @__PURE__ */ jsxDEV19("text", {
6187
+ fg: theme.success,
6188
+ children: " \u2192 Higher is better (faster streaming)"
6189
+ }, undefined, false, undefined, this),
6190
+ /* @__PURE__ */ jsxDEV19("text", {
6191
+ fg: theme.text,
6192
+ children: " "
6193
+ }, undefined, false, undefined, this),
6194
+ /* @__PURE__ */ jsxDEV19("text", {
6195
+ fg: theme.secondary,
6196
+ bold: true,
6197
+ children: "TTFT (Time To First Token)"
6198
+ }, undefined, false, undefined, this),
6199
+ /* @__PURE__ */ jsxDEV19("text", {
6200
+ fg: theme.text,
6201
+ children: " Time from request to receiving the first token."
6202
+ }, undefined, false, undefined, this),
6203
+ /* @__PURE__ */ jsxDEV19("text", {
6204
+ fg: theme.dim,
6205
+ children: " Formula: first_token_time - request_start_time"
6206
+ }, undefined, false, undefined, this),
6207
+ /* @__PURE__ */ jsxDEV19("text", {
6208
+ fg: theme.success,
6209
+ children: " \u2192 Lower is better (less waiting)"
6210
+ }, undefined, false, undefined, this),
6211
+ /* @__PURE__ */ jsxDEV19("text", {
6212
+ fg: theme.text,
6213
+ children: " "
6214
+ }, undefined, false, undefined, this),
6215
+ /* @__PURE__ */ jsxDEV19("text", {
6216
+ fg: theme.secondary,
6217
+ bold: true,
6218
+ children: "F1000 (First to 1000)"
6219
+ }, undefined, false, undefined, this),
6220
+ /* @__PURE__ */ jsxDEV19("text", {
6221
+ fg: theme.text,
6222
+ children: " Time to complete 1000 agentic requests (~300 tokens each)."
6223
+ }, undefined, false, undefined, this),
6224
+ /* @__PURE__ */ jsxDEV19("text", {
6225
+ fg: theme.dim,
6226
+ children: " Formula: 1000 \xD7 (TTFT + 300/TPS) = hours"
6227
+ }, undefined, false, undefined, this),
6228
+ /* @__PURE__ */ jsxDEV19("text", {
6229
+ fg: theme.success,
6230
+ children: " \u2192 Lower is better"
6231
+ }, undefined, false, undefined, this),
6232
+ /* @__PURE__ */ jsxDEV19("text", {
6233
+ fg: theme.text,
6234
+ children: " "
6235
+ }, undefined, false, undefined, this),
6236
+ /* @__PURE__ */ jsxDEV19("text", {
6237
+ fg: theme.text,
6238
+ children: " Why it matters: In agentic coding (Cursor, Copilot, OpenCode),"
6239
+ }, undefined, false, undefined, this),
6240
+ /* @__PURE__ */ jsxDEV19("text", {
6241
+ fg: theme.text,
6242
+ children: " models make hundreds of tool calls \u2014 TTFT adds up massively."
6243
+ }, undefined, false, undefined, this),
6244
+ /* @__PURE__ */ jsxDEV19("text", {
6245
+ fg: theme.text,
6246
+ children: " A 30 tok/s + 1s TTFT model can match a 60 tok/s + 6s TTFT model."
6247
+ }, undefined, false, undefined, this),
6248
+ /* @__PURE__ */ jsxDEV19("text", {
6249
+ fg: theme.text,
6250
+ children: " "
6251
+ }, undefined, false, undefined, this),
6252
+ /* @__PURE__ */ jsxDEV19("text", {
6253
+ fg: theme.secondary,
6254
+ bold: true,
6255
+ children: "Total Time"
6256
+ }, undefined, false, undefined, this),
6257
+ /* @__PURE__ */ jsxDEV19("text", {
6258
+ fg: theme.text,
6259
+ children: " Complete request duration from start to finish."
6260
+ }, undefined, false, undefined, this),
6261
+ /* @__PURE__ */ jsxDEV19("text", {
6262
+ fg: theme.dim,
6263
+ children: " Includes: connection, TTFT, and generation time"
6264
+ }, undefined, false, undefined, this)
6265
+ ]
6266
+ }, undefined, true, undefined, this),
6267
+ /* @__PURE__ */ jsxDEV19("box", {
6268
+ flexDirection: "column",
6269
+ border: true,
6270
+ borderStyle: "rounded",
6271
+ borderColor: theme.border,
6272
+ padding: 1,
6273
+ marginBottom: 1,
6274
+ children: [
6275
+ /* @__PURE__ */ jsxDEV19("text", {
6276
+ fg: theme.accent,
6277
+ bold: true,
6278
+ children: "LINKS"
6279
+ }, undefined, false, undefined, this),
6280
+ /* @__PURE__ */ jsxDEV19("text", {
6281
+ fg: theme.text,
6282
+ children: " "
6283
+ }, undefined, false, undefined, this),
6284
+ /* @__PURE__ */ jsxDEV19("text", {
6285
+ fg: theme.secondary,
6286
+ bold: true,
6287
+ children: "Discord Community"
6288
+ }, undefined, false, undefined, this),
6289
+ /* @__PURE__ */ jsxDEV19("text", {
6290
+ fg: theme.text,
6291
+ children: " https://discord.gg/6S7HwCxbMy"
6292
+ }, undefined, false, undefined, this),
6293
+ /* @__PURE__ */ jsxDEV19("text", {
6294
+ fg: theme.dim,
6295
+ children: " Join for help, updates, and discussions"
6296
+ }, undefined, false, undefined, this),
6297
+ /* @__PURE__ */ jsxDEV19("text", {
6298
+ fg: theme.text,
6299
+ children: " "
6300
+ }, undefined, false, undefined, this),
6301
+ /* @__PURE__ */ jsxDEV19("text", {
6302
+ fg: theme.secondary,
6303
+ bold: true,
6304
+ children: "Website & Leaderboard"
6305
+ }, undefined, false, undefined, this),
6306
+ /* @__PURE__ */ jsxDEV19("text", {
6307
+ fg: theme.text,
6308
+ children: " https://ai-speedometer.oliveowl.xyz/"
6309
+ }, undefined, false, undefined, this),
6310
+ /* @__PURE__ */ jsxDEV19("text", {
6311
+ fg: theme.dim,
6312
+ children: " Track OSS model speeds over time"
6313
+ }, undefined, false, undefined, this),
6314
+ /* @__PURE__ */ jsxDEV19("text", {
6315
+ fg: theme.text,
6316
+ children: " "
6317
+ }, undefined, false, undefined, this),
6318
+ /* @__PURE__ */ jsxDEV19("text", {
6319
+ fg: theme.secondary,
6320
+ bold: true,
6321
+ children: "GitHub"
6322
+ }, undefined, false, undefined, this),
6323
+ /* @__PURE__ */ jsxDEV19("text", {
6324
+ fg: theme.text,
6325
+ children: " https://github.com/anomaly/ai-speedometer"
6326
+ }, undefined, false, undefined, this),
6327
+ /* @__PURE__ */ jsxDEV19("text", {
6328
+ fg: theme.dim,
6329
+ children: " Source code, issues, contributions"
6330
+ }, undefined, false, undefined, this)
6331
+ ]
6332
+ }, undefined, true, undefined, this),
6333
+ /* @__PURE__ */ jsxDEV19("box", {
6334
+ flexDirection: "column",
6335
+ border: true,
6336
+ borderStyle: "rounded",
6337
+ borderColor: theme.border,
6338
+ padding: 1,
6339
+ marginBottom: 1,
6340
+ children: [
6341
+ /* @__PURE__ */ jsxDEV19("text", {
6342
+ fg: theme.accent,
6343
+ bold: true,
6344
+ children: "TIPS"
6345
+ }, undefined, false, undefined, this),
6346
+ /* @__PURE__ */ jsxDEV19("text", {
6347
+ fg: theme.text,
6348
+ children: " "
6349
+ }, undefined, false, undefined, this),
6350
+ /* @__PURE__ */ jsxDEV19("text", {
6351
+ fg: theme.text,
6352
+ children: " \u2022 Press [T] anytime to open the theme picker"
6353
+ }, undefined, false, undefined, this),
6354
+ /* @__PURE__ */ jsxDEV19("text", {
6355
+ fg: theme.text,
6356
+ children: " \u2022 Use --log flag to save raw SSE data for debugging"
6357
+ }, undefined, false, undefined, this),
6358
+ /* @__PURE__ */ jsxDEV19("text", {
6359
+ fg: theme.text,
6360
+ children: " \u2022 Run headless with ai-speedometer-headless for CI/CD"
6361
+ }, undefined, false, undefined, this),
6362
+ /* @__PURE__ */ jsxDEV19("text", {
6363
+ fg: theme.text,
6364
+ children: " \u2022 [*] in results means token count was estimated"
6365
+ }, undefined, false, undefined, this)
6366
+ ]
6367
+ }, undefined, true, undefined, this),
6368
+ /* @__PURE__ */ jsxDEV19("box", {
6369
+ flexDirection: "row",
6370
+ justifyContent: "center",
6371
+ marginTop: 1,
6372
+ children: /* @__PURE__ */ jsxDEV19("text", {
6373
+ fg: theme.dim,
6374
+ children: "Press [Q] or [Esc] to return to main menu"
6375
+ }, undefined, false, undefined, this)
6376
+ }, undefined, false, undefined, this)
6377
+ ]
6378
+ }, undefined, true, undefined, this)
6379
+ }, undefined, false, undefined, this)
6380
+ ]
6381
+ }, undefined, true, undefined, this)
6382
+ }, undefined, false, undefined, this);
6383
+ }
6384
+ var init_FAQScreen = __esm(() => {
6385
+ init_useAppKeyboard();
6386
+ init_AppContext();
6387
+ init_ThemeContext();
6388
+ });
6389
+
5936
6390
  // src/tui/App.tsx
5937
6391
  import { useKeyboard as useKeyboard3, useRenderer as useRenderer3 } from "@opentui/react";
5938
- import { jsxDEV as jsxDEV19 } from "@opentui/react/jsx-dev-runtime";
6392
+ import { jsxDEV as jsxDEV20 } from "@opentui/react/jsx-dev-runtime";
5939
6393
  function getHints(screen, benchResults) {
5940
6394
  switch (screen) {
5941
6395
  case "main-menu":
@@ -5946,10 +6400,12 @@ function getHints(screen, benchResults) {
5946
6400
  return ["[\u2191\u2193] navigate", "[Tab] select", "[Enter] run", "[A] all", "[N] none", "[R] recent", "[Esc] back"];
5947
6401
  case "benchmark": {
5948
6402
  const allDone = benchResults.length > 0 && benchResults.every((r) => r.status === "done" || r.status === "error");
5949
- return allDone ? ["[Enter] back to menu", "[Q] back to menu"] : ["Benchmark in progress..."];
6403
+ return allDone ? ["[R] rerun", "[Enter] back to menu", "[Q] back to menu"] : ["Benchmark in progress..."];
5950
6404
  }
5951
6405
  case "list-providers":
5952
6406
  return ["[\u2191\u2193] scroll", "[Q] back"];
6407
+ case "faq":
6408
+ return ["[\u2191\u2193] scroll", "[Q] back"];
5953
6409
  case "add-verified":
5954
6410
  return ["[\u2191\u2193] navigate", "[Enter] select", "[Q] back"];
5955
6411
  case "add-custom":
@@ -5964,21 +6420,23 @@ function ActiveScreen() {
5964
6420
  const { state } = useAppContext();
5965
6421
  switch (state.screen) {
5966
6422
  case "main-menu":
5967
- return /* @__PURE__ */ jsxDEV19(MainMenuScreen, {}, undefined, false, undefined, this);
6423
+ return /* @__PURE__ */ jsxDEV20(MainMenuScreen, {}, undefined, false, undefined, this);
5968
6424
  case "model-menu":
5969
- return /* @__PURE__ */ jsxDEV19(ModelMenuScreen, {}, undefined, false, undefined, this);
6425
+ return /* @__PURE__ */ jsxDEV20(ModelMenuScreen, {}, undefined, false, undefined, this);
5970
6426
  case "model-select":
5971
- return /* @__PURE__ */ jsxDEV19(ModelSelectScreen, {}, undefined, false, undefined, this);
6427
+ return /* @__PURE__ */ jsxDEV20(ModelSelectScreen, {}, undefined, false, undefined, this);
5972
6428
  case "benchmark":
5973
- return /* @__PURE__ */ jsxDEV19(BenchmarkScreen, {}, undefined, false, undefined, this);
6429
+ return /* @__PURE__ */ jsxDEV20(BenchmarkScreen, {}, undefined, false, undefined, this);
5974
6430
  case "add-verified":
5975
- return /* @__PURE__ */ jsxDEV19(AddVerifiedScreen, {}, undefined, false, undefined, this);
6431
+ return /* @__PURE__ */ jsxDEV20(AddVerifiedScreen, {}, undefined, false, undefined, this);
5976
6432
  case "add-custom":
5977
- return /* @__PURE__ */ jsxDEV19(AddCustomScreen, {}, undefined, false, undefined, this);
6433
+ return /* @__PURE__ */ jsxDEV20(AddCustomScreen, {}, undefined, false, undefined, this);
5978
6434
  case "add-models":
5979
- return /* @__PURE__ */ jsxDEV19(AddModelsScreen, {}, undefined, false, undefined, this);
6435
+ return /* @__PURE__ */ jsxDEV20(AddModelsScreen, {}, undefined, false, undefined, this);
5980
6436
  case "list-providers":
5981
- return /* @__PURE__ */ jsxDEV19(ListProvidersScreen, {}, undefined, false, undefined, this);
6437
+ return /* @__PURE__ */ jsxDEV20(ListProvidersScreen, {}, undefined, false, undefined, this);
6438
+ case "faq":
6439
+ return /* @__PURE__ */ jsxDEV20(FAQScreen, {}, undefined, false, undefined, this);
5982
6440
  }
5983
6441
  }
5984
6442
  function Shell() {
@@ -5995,36 +6453,36 @@ function Shell() {
5995
6453
  setModalOpen(!modalOpen);
5996
6454
  }
5997
6455
  });
5998
- return /* @__PURE__ */ jsxDEV19("box", {
6456
+ return /* @__PURE__ */ jsxDEV20("box", {
5999
6457
  flexDirection: "column",
6000
6458
  height: "100%",
6001
6459
  width: "100%",
6002
6460
  backgroundColor: theme.background,
6003
6461
  children: [
6004
- /* @__PURE__ */ jsxDEV19(Header, {
6462
+ /* @__PURE__ */ jsxDEV20(Header, {
6005
6463
  screen: state.screen
6006
6464
  }, undefined, false, undefined, this),
6007
- /* @__PURE__ */ jsxDEV19("box", {
6465
+ /* @__PURE__ */ jsxDEV20("box", {
6008
6466
  flexGrow: 1,
6009
6467
  flexDirection: "column",
6010
- children: /* @__PURE__ */ jsxDEV19(ActiveScreen, {}, undefined, false, undefined, this)
6468
+ children: /* @__PURE__ */ jsxDEV20(ActiveScreen, {}, undefined, false, undefined, this)
6011
6469
  }, undefined, false, undefined, this),
6012
- /* @__PURE__ */ jsxDEV19(Footer, {
6470
+ /* @__PURE__ */ jsxDEV20(Footer, {
6013
6471
  hints: getHints(state.screen, state.benchResults)
6014
6472
  }, undefined, false, undefined, this),
6015
- modalOpen && /* @__PURE__ */ jsxDEV19(ThemePicker, {
6473
+ modalOpen && /* @__PURE__ */ jsxDEV20(ThemePicker, {
6016
6474
  onClose: () => setModalOpen(false)
6017
6475
  }, undefined, false, undefined, this)
6018
6476
  ]
6019
6477
  }, undefined, true, undefined, this);
6020
6478
  }
6021
6479
  function App({ logMode = false, theme = "tokyonight" }) {
6022
- return /* @__PURE__ */ jsxDEV19(ThemeProvider, {
6480
+ return /* @__PURE__ */ jsxDEV20(ThemeProvider, {
6023
6481
  name: theme,
6024
- children: /* @__PURE__ */ jsxDEV19(ModalProvider, {
6025
- children: /* @__PURE__ */ jsxDEV19(AppProvider, {
6482
+ children: /* @__PURE__ */ jsxDEV20(ModalProvider, {
6483
+ children: /* @__PURE__ */ jsxDEV20(AppProvider, {
6026
6484
  logMode,
6027
- children: /* @__PURE__ */ jsxDEV19(Shell, {}, undefined, false, undefined, this)
6485
+ children: /* @__PURE__ */ jsxDEV20(Shell, {}, undefined, false, undefined, this)
6028
6486
  }, undefined, false, undefined, this)
6029
6487
  }, undefined, false, undefined, this)
6030
6488
  }, undefined, false, undefined, this);
@@ -6044,6 +6502,7 @@ var init_App = __esm(() => {
6044
6502
  init_AddCustomScreen();
6045
6503
  init_AddModelsScreen();
6046
6504
  init_ListProvidersScreen();
6505
+ init_FAQScreen();
6047
6506
  });
6048
6507
 
6049
6508
  // src/tui/index.tsx
@@ -6053,7 +6512,7 @@ __export(exports_tui, {
6053
6512
  });
6054
6513
  import { createCliRenderer } from "@opentui/core";
6055
6514
  import { createRoot } from "@opentui/react";
6056
- import { jsxDEV as jsxDEV20 } from "@opentui/react/jsx-dev-runtime";
6515
+ import { jsxDEV as jsxDEV21 } from "@opentui/react/jsx-dev-runtime";
6057
6516
  async function startTui(logMode = false) {
6058
6517
  const { readThemeFromConfig: readThemeFromConfig2 } = await Promise.resolve().then(() => (init_ai_config(), exports_ai_config));
6059
6518
  const theme = await readThemeFromConfig2();
@@ -6070,7 +6529,7 @@ async function startTui(logMode = false) {
6070
6529
  renderer.destroy();
6071
6530
  process.exit(0);
6072
6531
  });
6073
- createRoot(renderer).render(/* @__PURE__ */ jsxDEV20(App, {
6532
+ createRoot(renderer).render(/* @__PURE__ */ jsxDEV21(App, {
6074
6533
  logMode,
6075
6534
  theme
6076
6535
  }, undefined, false, undefined, this));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-speedometer",
3
- "version": "2.2.1",
3
+ "version": "2.3.1",
4
4
  "description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
5
5
  "bin": {
6
6
  "ai-speedometer": "dist/ai-speedometer",