ai-speedometer 2.1.2 → 2.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-speedometer +21 -26
- package/package.json +8 -13
- package/README.md +0 -127
- package/ai-benchmark-config.json.template +0 -21
- package/dist/ai-speedometer-headless +0 -941
- package/docs/README.md +0 -191
- package/docs/custom-verified-providers.md +0 -463
- package/docs/models-dev-integration.md +0 -488
- package/docs/publish-and-build.md +0 -68
- package/docs/token-counting-fallback.md +0 -345
- package/scripts/shebang +0 -6
package/dist/ai-speedometer
CHANGED
|
@@ -17,7 +17,7 @@ var __export = (target, all) => {
|
|
|
17
17
|
};
|
|
18
18
|
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
19
19
|
|
|
20
|
-
// custom-verified-providers.json
|
|
20
|
+
// ../core/custom-verified-providers.json
|
|
21
21
|
var custom_verified_providers_default;
|
|
22
22
|
var init_custom_verified_providers = __esm(() => {
|
|
23
23
|
custom_verified_providers_default = {
|
|
@@ -142,7 +142,7 @@ var init_custom_verified_providers = __esm(() => {
|
|
|
142
142
|
};
|
|
143
143
|
});
|
|
144
144
|
|
|
145
|
-
// src/models-dev.ts
|
|
145
|
+
// ../core/src/models-dev.ts
|
|
146
146
|
var exports_models_dev = {};
|
|
147
147
|
__export(exports_models_dev, {
|
|
148
148
|
transformModelsDevData: () => transformModelsDevData,
|
|
@@ -387,7 +387,7 @@ var init_models_dev = __esm(() => {
|
|
|
387
387
|
];
|
|
388
388
|
});
|
|
389
389
|
|
|
390
|
-
// src/ai-config.ts
|
|
390
|
+
// ../core/src/ai-config.ts
|
|
391
391
|
var exports_ai_config = {};
|
|
392
392
|
__export(exports_ai_config, {
|
|
393
393
|
writeAIConfig: () => writeAIConfig,
|
|
@@ -558,7 +558,7 @@ var getAIConfigPaths = () => {
|
|
|
558
558
|
};
|
|
559
559
|
var init_ai_config = () => {};
|
|
560
560
|
|
|
561
|
-
// node_modules/jsonc-parser/lib/esm/impl/scanner.js
|
|
561
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/impl/scanner.js
|
|
562
562
|
function createScanner(text, ignoreTrivia = false) {
|
|
563
563
|
const len = text.length;
|
|
564
564
|
let pos = 0, value = "", tokenOffset = 0, token = 16, lineNumber = 0, lineStartOffset = 0, tokenLineStartOffset = 0, prevTokenLineStartOffset = 0, scanError = 0;
|
|
@@ -975,7 +975,7 @@ var init_scanner = __esm(() => {
|
|
|
975
975
|
})(CharacterCodes || (CharacterCodes = {}));
|
|
976
976
|
});
|
|
977
977
|
|
|
978
|
-
// node_modules/jsonc-parser/lib/esm/impl/string-intern.js
|
|
978
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/impl/string-intern.js
|
|
979
979
|
var cachedSpaces, maxCachedValues = 200, cachedBreakLinesWithSpaces;
|
|
980
980
|
var init_string_intern = __esm(() => {
|
|
981
981
|
cachedSpaces = new Array(20).fill(0).map((_, index) => {
|
|
@@ -1011,13 +1011,13 @@ var init_string_intern = __esm(() => {
|
|
|
1011
1011
|
};
|
|
1012
1012
|
});
|
|
1013
1013
|
|
|
1014
|
-
// node_modules/jsonc-parser/lib/esm/impl/format.js
|
|
1014
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/impl/format.js
|
|
1015
1015
|
var init_format = __esm(() => {
|
|
1016
1016
|
init_scanner();
|
|
1017
1017
|
init_string_intern();
|
|
1018
1018
|
});
|
|
1019
1019
|
|
|
1020
|
-
// node_modules/jsonc-parser/lib/esm/impl/parser.js
|
|
1020
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/impl/parser.js
|
|
1021
1021
|
function parse(text, errors = [], options = ParseOptions.DEFAULT) {
|
|
1022
1022
|
let currentProperty = null;
|
|
1023
1023
|
let currentParent = [];
|
|
@@ -1323,13 +1323,13 @@ var init_parser = __esm(() => {
|
|
|
1323
1323
|
})(ParseOptions || (ParseOptions = {}));
|
|
1324
1324
|
});
|
|
1325
1325
|
|
|
1326
|
-
// node_modules/jsonc-parser/lib/esm/impl/edit.js
|
|
1326
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/impl/edit.js
|
|
1327
1327
|
var init_edit = __esm(() => {
|
|
1328
1328
|
init_format();
|
|
1329
1329
|
init_parser();
|
|
1330
1330
|
});
|
|
1331
1331
|
|
|
1332
|
-
// node_modules/jsonc-parser/lib/esm/main.js
|
|
1332
|
+
// ../../node_modules/.bun/jsonc-parser@3.3.1/node_modules/jsonc-parser/lib/esm/main.js
|
|
1333
1333
|
var ScanError, SyntaxKind, parse2, ParseErrorCode;
|
|
1334
1334
|
var init_main = __esm(() => {
|
|
1335
1335
|
init_format();
|
|
@@ -1385,7 +1385,7 @@ var init_main = __esm(() => {
|
|
|
1385
1385
|
})(ParseErrorCode || (ParseErrorCode = {}));
|
|
1386
1386
|
});
|
|
1387
1387
|
|
|
1388
|
-
// src/opencode-integration.ts
|
|
1388
|
+
// ../core/src/opencode-integration.ts
|
|
1389
1389
|
var exports_opencode_integration = {};
|
|
1390
1390
|
__export(exports_opencode_integration, {
|
|
1391
1391
|
writeOpencodeConfig: () => writeOpencodeConfig,
|
|
@@ -1704,10 +1704,10 @@ var init_opencode_integration = __esm(() => {
|
|
|
1704
1704
|
init_main();
|
|
1705
1705
|
});
|
|
1706
1706
|
|
|
1707
|
-
// src/constants.ts
|
|
1707
|
+
// ../core/src/constants.ts
|
|
1708
1708
|
var TEST_PROMPT = `make a 300 word story`;
|
|
1709
1709
|
|
|
1710
|
-
// src/benchmark.ts
|
|
1710
|
+
// ../core/src/benchmark.ts
|
|
1711
1711
|
var exports_benchmark = {};
|
|
1712
1712
|
__export(exports_benchmark, {
|
|
1713
1713
|
benchmarkSingleModelRest: () => benchmarkSingleModelRest
|
|
@@ -1904,7 +1904,7 @@ async function benchmarkSingleModelRest(model) {
|
|
|
1904
1904
|
}
|
|
1905
1905
|
var init_benchmark = () => {};
|
|
1906
1906
|
|
|
1907
|
-
// src/headless.ts
|
|
1907
|
+
// ../core/src/headless.ts
|
|
1908
1908
|
var exports_headless = {};
|
|
1909
1909
|
__export(exports_headless, {
|
|
1910
1910
|
runHeadlessBenchmark: () => runHeadlessBenchmark
|
|
@@ -2161,25 +2161,20 @@ var package_default;
|
|
|
2161
2161
|
var init_package = __esm(() => {
|
|
2162
2162
|
package_default = {
|
|
2163
2163
|
name: "ai-speedometer",
|
|
2164
|
-
version: "2.1.
|
|
2164
|
+
version: "2.1.4",
|
|
2165
2165
|
description: "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
|
|
2166
2166
|
bin: {
|
|
2167
2167
|
"ai-speedometer": "dist/ai-speedometer",
|
|
2168
|
-
aispeed: "dist/ai-speedometer"
|
|
2169
|
-
"ai-speedometer-headless": "dist/ai-speedometer-headless"
|
|
2168
|
+
aispeed: "dist/ai-speedometer"
|
|
2170
2169
|
},
|
|
2171
2170
|
engines: {
|
|
2172
|
-
bun: ">=1.0.0"
|
|
2173
|
-
node: ">=18.0.0"
|
|
2171
|
+
bun: ">=1.0.0"
|
|
2174
2172
|
},
|
|
2175
2173
|
scripts: {
|
|
2176
2174
|
start: "bun src/index.ts",
|
|
2177
2175
|
dev: "bun --watch src/index.ts",
|
|
2178
|
-
|
|
2179
|
-
"test:watch": "bun test --watch",
|
|
2180
|
-
"test:update": "bun test --update-snapshots",
|
|
2176
|
+
build: "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat ../../scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js",
|
|
2181
2177
|
typecheck: "bun tsc --noEmit",
|
|
2182
|
-
build: "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js && bun build src/headless-entry.ts --outdir dist --target node --external 'jsonc-parser' && mv dist/headless-entry.js dist/ai-speedometer-headless && chmod +x dist/ai-speedometer-headless",
|
|
2183
2178
|
prepublishOnly: "bun run build"
|
|
2184
2179
|
},
|
|
2185
2180
|
keywords: [
|
|
@@ -2209,10 +2204,9 @@ var init_package = __esm(() => {
|
|
|
2209
2204
|
homepage: "https://github.com/aptdnfapt/Ai-speedometer#readme",
|
|
2210
2205
|
files: [
|
|
2211
2206
|
"dist/",
|
|
2212
|
-
"docs/",
|
|
2213
|
-
"
|
|
2214
|
-
"
|
|
2215
|
-
"ai-benchmark-config.json.template"
|
|
2207
|
+
"../../docs/",
|
|
2208
|
+
"../../README.md",
|
|
2209
|
+
"../../ai-benchmark-config.json.template"
|
|
2216
2210
|
],
|
|
2217
2211
|
dependencies: {
|
|
2218
2212
|
"@opentui/core": "0.1.79",
|
|
@@ -2221,6 +2215,7 @@ var init_package = __esm(() => {
|
|
|
2221
2215
|
"react-reconciler": "^0.32.0"
|
|
2222
2216
|
},
|
|
2223
2217
|
devDependencies: {
|
|
2218
|
+
"@ai-speedometer/core": "workspace:*",
|
|
2224
2219
|
"jsonc-parser": "^3.3.1"
|
|
2225
2220
|
}
|
|
2226
2221
|
};
|
package/package.json
CHANGED
|
@@ -1,24 +1,19 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-speedometer",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.4",
|
|
4
4
|
"description": "A comprehensive CLI tool for benchmarking AI models across multiple providers with parallel execution and professional metrics",
|
|
5
5
|
"bin": {
|
|
6
6
|
"ai-speedometer": "dist/ai-speedometer",
|
|
7
|
-
"aispeed": "dist/ai-speedometer"
|
|
8
|
-
"ai-speedometer-headless": "dist/ai-speedometer-headless"
|
|
7
|
+
"aispeed": "dist/ai-speedometer"
|
|
9
8
|
},
|
|
10
9
|
"engines": {
|
|
11
|
-
"bun": ">=1.0.0"
|
|
12
|
-
"node": ">=18.0.0"
|
|
10
|
+
"bun": ">=1.0.0"
|
|
13
11
|
},
|
|
14
12
|
"scripts": {
|
|
15
13
|
"start": "bun src/index.ts",
|
|
16
14
|
"dev": "bun --watch src/index.ts",
|
|
17
|
-
"
|
|
18
|
-
"test:watch": "bun test --watch",
|
|
19
|
-
"test:update": "bun test --update-snapshots",
|
|
15
|
+
"build": "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat ../../scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js",
|
|
20
16
|
"typecheck": "bun tsc --noEmit",
|
|
21
|
-
"build": "bun build src/index.ts --outdir dist --target bun --external '@opentui/core' --external '@opentui/react' --external 'react' --external 'react-reconciler' && cat scripts/shebang dist/index.js > dist/ai-speedometer && chmod +x dist/ai-speedometer && rm dist/index.js && bun build src/headless-entry.ts --outdir dist --target node --external 'jsonc-parser' && mv dist/headless-entry.js dist/ai-speedometer-headless && chmod +x dist/ai-speedometer-headless",
|
|
22
17
|
"prepublishOnly": "bun run build"
|
|
23
18
|
},
|
|
24
19
|
"keywords": [
|
|
@@ -48,10 +43,9 @@
|
|
|
48
43
|
"homepage": "https://github.com/aptdnfapt/Ai-speedometer#readme",
|
|
49
44
|
"files": [
|
|
50
45
|
"dist/",
|
|
51
|
-
"docs/",
|
|
52
|
-
"
|
|
53
|
-
"
|
|
54
|
-
"ai-benchmark-config.json.template"
|
|
46
|
+
"../../docs/",
|
|
47
|
+
"../../README.md",
|
|
48
|
+
"../../ai-benchmark-config.json.template"
|
|
55
49
|
],
|
|
56
50
|
"dependencies": {
|
|
57
51
|
"@opentui/core": "0.1.79",
|
|
@@ -60,6 +54,7 @@
|
|
|
60
54
|
"react-reconciler": "^0.32.0"
|
|
61
55
|
},
|
|
62
56
|
"devDependencies": {
|
|
57
|
+
"@ai-speedometer/core": "workspace:*",
|
|
63
58
|
"jsonc-parser": "^3.3.1"
|
|
64
59
|
}
|
|
65
60
|
}
|
package/README.md
DELETED
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
# Ai-speedometer
|
|
2
|
-
|
|
3
|
-
A CLI tool for benchmarking AI models across multiple providers with parallel execution and performance metrics.
|
|
4
|
-
|
|
5
|
-
[](https://discord.gg/6S7HwCxbMy)
|
|
6
|
-
|
|
7
|
-
Track OSS model speeds over time: [ai-speedometer.oliveowl.xyz](https://ai-speedometer.oliveowl.xyz/)
|
|
8
|
-
|
|
9
|
-

|
|
10
|
-
|
|
11
|
-

|
|
12
|
-
|
|
13
|
-
## Install
|
|
14
|
-
|
|
15
|
-
Requires [Bun](https://bun.sh) runtime.
|
|
16
|
-
|
|
17
|
-
```bash
|
|
18
|
-
bun install -g ai-speedometer
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
Or with npm (Bun still required at runtime):
|
|
22
|
-
|
|
23
|
-
```bash
|
|
24
|
-
npm install -g ai-speedometer
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
Or run directly from source:
|
|
28
|
-
|
|
29
|
-
```bash
|
|
30
|
-
bun src/index.ts
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## What It Measures
|
|
34
|
-
|
|
35
|
-
- **TTFT** (Time to First Token) - How fast the first response token arrives
|
|
36
|
-
- **Total Time** - Complete request duration
|
|
37
|
-
- **Tokens/Second** - Real-time throughput
|
|
38
|
-
- **Token Counts** - Input, output, and total tokens used
|
|
39
|
-
|
|
40
|
-
## Features
|
|
41
|
-
|
|
42
|
-
- **Interactive TUI** - Full terminal UI with Tokyo Night theme, menus, search, and live benchmark progress
|
|
43
|
-
- **REST API Benchmarking** - Default method, works with all OpenAI-compatible providers
|
|
44
|
-
- **Headless Mode** - Run benchmarks without interactive CLI using command-line arguments
|
|
45
|
-
- **Parallel Execution** - Benchmark multiple models simultaneously
|
|
46
|
-
- **Provider Management** - Add verified, custom verified, and custom providers
|
|
47
|
-
|
|
48
|
-
## Quick Setup
|
|
49
|
-
|
|
50
|
-
1. **Set Model**
|
|
51
|
-
```bash
|
|
52
|
-
ai-speedometer
|
|
53
|
-
# Select "Run Benchmark" → "Add Verified Provider" → Choose provider (OpenAI, Anthropic, etc.)
|
|
54
|
-
# Enter your API key when prompted
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
2. **Choose Model Provider**
|
|
58
|
-
- Verified providers (OpenAI, Anthropic, Google) - auto-configured via models.dev
|
|
59
|
-
- Custom verified providers (pre-configured trusted providers) - add API key
|
|
60
|
-
- Custom providers (Ollama, local models) - add your base URL
|
|
61
|
-
|
|
62
|
-
3. **Add API Key**
|
|
63
|
-
- Get API keys from your provider's dashboard
|
|
64
|
-
- Enter when prompted - stored securely in:
|
|
65
|
-
- `~/.local/share/opencode/auth.json` (primary storage)
|
|
66
|
-
- `~/.config/ai-speedometer/ai-benchmark-config.json` (backup storage)
|
|
67
|
-
|
|
68
|
-
4. **Run Benchmark**
|
|
69
|
-
```bash
|
|
70
|
-
ai-speedometer
|
|
71
|
-
# Select "Run Benchmark" → choose models → press Enter
|
|
72
|
-
```
|
|
73
|
-
|
|
74
|
-
## Usage
|
|
75
|
-
|
|
76
|
-
```bash
|
|
77
|
-
# Start interactive TUI
|
|
78
|
-
ai-speedometer
|
|
79
|
-
|
|
80
|
-
# Short alias
|
|
81
|
-
aispeed
|
|
82
|
-
|
|
83
|
-
# Debug mode
|
|
84
|
-
ai-speedometer --debug
|
|
85
|
-
|
|
86
|
-
# Headless benchmark
|
|
87
|
-
ai-speedometer --bench openai:gpt-4
|
|
88
|
-
# With custom API key
|
|
89
|
-
ai-speedometer --bench openai:gpt-4 --api-key "sk-your-key"
|
|
90
|
-
# Custom provider
|
|
91
|
-
ai-speedometer --bench-custom myprovider:mymodel --base-url https://... --api-key "..."
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
## Development
|
|
95
|
-
|
|
96
|
-
```bash
|
|
97
|
-
# Run from source
|
|
98
|
-
bun src/index.ts
|
|
99
|
-
|
|
100
|
-
# Run with auto-reload
|
|
101
|
-
bun --watch src/index.ts
|
|
102
|
-
|
|
103
|
-
# Run tests
|
|
104
|
-
bun test
|
|
105
|
-
|
|
106
|
-
# Typecheck
|
|
107
|
-
bun run typecheck
|
|
108
|
-
|
|
109
|
-
# Build standalone binary
|
|
110
|
-
bun run build # → dist/ai-speedometer
|
|
111
|
-
```
|
|
112
|
-
|
|
113
|
-
## Configuration Files
|
|
114
|
-
|
|
115
|
-
API keys and configuration are stored in:
|
|
116
|
-
|
|
117
|
-
- **Verified + Custom Verified Providers**:
|
|
118
|
-
- Primary: `~/.local/share/opencode/auth.json`
|
|
119
|
-
- Backup: `~/.config/ai-speedometer/ai-benchmark-config.json` (verifiedProviders section)
|
|
120
|
-
- **Custom Providers**: `~/.config/ai-speedometer/ai-benchmark-config.json` (customProviders section)
|
|
121
|
-
- **Provider Definitions**: `./custom-verified-providers.json` (bundled at build time)
|
|
122
|
-
|
|
123
|
-
## Requirements
|
|
124
|
-
|
|
125
|
-
- **Runtime**: Bun 1.0+ (required — install from [bun.sh](https://bun.sh))
|
|
126
|
-
- API keys for AI providers
|
|
127
|
-
- Terminal with arrow keys and ANSI colors
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"verifiedProviders": {
|
|
3
|
-
"openai": "sk-your-openai-api-key",
|
|
4
|
-
"anthropic": "sk-ant-your-anthropic-api-key"
|
|
5
|
-
},
|
|
6
|
-
"customProviders": [
|
|
7
|
-
{
|
|
8
|
-
"id": "my-custom-provider",
|
|
9
|
-
"name": "My Custom Provider",
|
|
10
|
-
"type": "openai-compatible",
|
|
11
|
-
"baseUrl": "https://api.custom.com/v1",
|
|
12
|
-
"apiKey": "YOUR_API_KEY_HERE",
|
|
13
|
-
"models": [
|
|
14
|
-
{
|
|
15
|
-
"name": "custom-model",
|
|
16
|
-
"id": "custom_model_1"
|
|
17
|
-
}
|
|
18
|
-
]
|
|
19
|
-
}
|
|
20
|
-
]
|
|
21
|
-
}
|