litmus-ai 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/cli.js +21 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ A terminal-based LLM benchmarking and evaluation tool built with **OpenTUI**. Co
|
|
|
6
6
|
|
|
7
7
|
## Features
|
|
8
8
|
|
|
9
|
-
###
|
|
9
|
+
### Model Comparison
|
|
10
10
|
|
|
11
11
|
- Run identical prompts across multiple LLMs simultaneously
|
|
12
12
|
- Real-time streaming responses with progress indicators
|
|
@@ -14,16 +14,17 @@ A terminal-based LLM benchmarking and evaluation tool built with **OpenTUI**. Co
|
|
|
14
14
|
- Visual comparison grid with response timing
|
|
15
15
|
- **Multi-modal support** - Attach images to prompts (see Image Attachments below)
|
|
16
16
|
|
|
17
|
-
|
|
18
17
|
### Image Attachments
|
|
19
18
|
|
|
20
19
|
Litmus supports multi-modal prompts with image attachments. You can attach images in multiple ways:
|
|
21
20
|
|
|
22
21
|
**Clipboard Paste (Ctrl+V)**
|
|
22
|
+
|
|
23
23
|
- Copy an image to your clipboard (Cmd/Ctrl+C on any image)
|
|
24
24
|
- Press `Ctrl+V` in the Benchmark view to attach
|
|
25
25
|
|
|
26
26
|
**File Path**
|
|
27
|
+
|
|
27
28
|
- Type or paste a file path to an image
|
|
28
29
|
- Supports `~/` home directory expansion
|
|
29
30
|
- Example: `~/photos/screenshot.png`
|
|
@@ -31,12 +32,12 @@ Litmus supports multi-modal prompts with image attachments. You can attach image
|
|
|
31
32
|
**Supported Formats**: PNG, JPG, JPEG, GIF, WebP, BMP
|
|
32
33
|
|
|
33
34
|
**Image Controls**
|
|
35
|
+
|
|
34
36
|
- `x` - Remove last attached image
|
|
35
37
|
- `c` - Clear all attached images
|
|
36
38
|
- `i` - Open image input dialog (alternative method)
|
|
37
39
|
- Images are displayed above the prompt input when attached
|
|
38
40
|
|
|
39
|
-
|
|
40
41
|
### Evals using LLM-as-Judge
|
|
41
42
|
|
|
42
43
|
- Run automated evaluations using dedicated judge models
|
|
@@ -54,12 +55,16 @@ Litmus supports multi-modal prompts with image attachments. You can attach image
|
|
|
54
55
|
|
|
55
56
|

|
|
56
57
|
|
|
58
|
+
## Installation
|
|
57
59
|
|
|
60
|
+
Requires [Bun](https://bun.sh)
|
|
58
61
|
|
|
59
|
-
|
|
62
|
+
```bash
|
|
63
|
+
bun add -g litmus-ai
|
|
64
|
+
```
|
|
60
65
|
|
|
61
66
|
```bash
|
|
62
|
-
|
|
67
|
+
litmus
|
|
63
68
|
```
|
|
64
69
|
|
|
65
70
|
### Environment Setup
|
|
@@ -93,7 +98,6 @@ OPENROUTER_API_KEY=your_key_here # Required - get from https://openrouter.ai
|
|
|
93
98
|
EXA_API_KEY=your_key_here # Optional - for web search tool (https://exa.ai)
|
|
94
99
|
```
|
|
95
100
|
|
|
96
|
-
|
|
97
101
|
## Evaluation Criteria
|
|
98
102
|
|
|
99
103
|
Litmus evaluates models on:
|
|
@@ -137,7 +141,6 @@ Litmus evaluates models on:
|
|
|
137
141
|
- **Enter** - Select run
|
|
138
142
|
- **Delete** - Remove run
|
|
139
143
|
|
|
140
|
-
|
|
141
144
|
## Development
|
|
142
145
|
|
|
143
146
|
```bash
|
package/dist/cli.js
CHANGED
|
@@ -80459,6 +80459,8 @@ function getAllToolNames() {
|
|
|
80459
80459
|
|
|
80460
80460
|
// src/db/index.ts
|
|
80461
80461
|
import { Database } from "bun:sqlite";
|
|
80462
|
+
import { mkdirSync } from "fs";
|
|
80463
|
+
import { dirname as dirname2, join as join2 } from "path";
|
|
80462
80464
|
|
|
80463
80465
|
// src/db/schema.ts
|
|
80464
80466
|
var SCHEMA = `
|
|
@@ -80768,9 +80770,27 @@ function listJudgeModels(db) {
|
|
|
80768
80770
|
|
|
80769
80771
|
// src/db/index.ts
|
|
80770
80772
|
var db = null;
|
|
80773
|
+
function getDefaultDbPath() {
|
|
80774
|
+
const override = process.env.LITMUS_DB_PATH;
|
|
80775
|
+
if (override && override.trim().length > 0) {
|
|
80776
|
+
return override;
|
|
80777
|
+
}
|
|
80778
|
+
const home = process.env.HOME ?? process.env.USERPROFILE ?? ".";
|
|
80779
|
+
if (process.platform === "darwin") {
|
|
80780
|
+
return join2(home, "Library", "Application Support", "litmus-ai", "Litmus.db");
|
|
80781
|
+
}
|
|
80782
|
+
if (process.platform === "win32") {
|
|
80783
|
+
const appData = process.env.APPDATA ?? join2(home, "AppData", "Roaming");
|
|
80784
|
+
return join2(appData, "litmus-ai", "Litmus.db");
|
|
80785
|
+
}
|
|
80786
|
+
const xdg = process.env.XDG_DATA_HOME ?? join2(home, ".local", "share");
|
|
80787
|
+
return join2(xdg, "litmus-ai", "Litmus.db");
|
|
80788
|
+
}
|
|
80771
80789
|
function getDatabase() {
|
|
80772
80790
|
if (!db) {
|
|
80773
|
-
|
|
80791
|
+
const dbPath = getDefaultDbPath();
|
|
80792
|
+
mkdirSync(dirname2(dbPath), { recursive: true });
|
|
80793
|
+
db = new Database(dbPath, { create: true });
|
|
80774
80794
|
db.exec("PRAGMA journal_mode = WAL");
|
|
80775
80795
|
db.exec("PRAGMA foreign_keys = ON");
|
|
80776
80796
|
db.exec(SCHEMA);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "litmus-ai",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.5",
|
|
4
4
|
"description": "Multi-model LLM eval CLI tool",
|
|
5
5
|
"module": "src/index.tsx",
|
|
6
6
|
"type": "module",
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
"dist"
|
|
12
12
|
],
|
|
13
13
|
"scripts": {
|
|
14
|
-
"dev": "bun run --watch src/index.tsx",
|
|
15
|
-
"build": "bun build src/index.tsx --outdir dist --target bun --entry-naming cli.js &&
|
|
14
|
+
"dev": "LITMUS_DB_PATH=./data/Litmus.db bun run --watch src/index.tsx",
|
|
15
|
+
"build": "bun build src/index.tsx --outdir dist --target bun --entry-naming cli.js && perl -0777 -pi -e 's/^#!.*\\n/#!\\/usr\\/bin\\/env bun\\n/' dist/cli.js",
|
|
16
16
|
"prepublishOnly": "bun run build"
|
|
17
17
|
},
|
|
18
18
|
"engines": {
|