evals 1.0.7 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -86
- package/cli.js +232 -0
- package/package.json +15 -31
- package/LICENSE +0 -7
- package/dist/index.js +0 -15372
- package/src/index.js +0 -190
package/README.md
CHANGED
|
@@ -1,103 +1,26 @@
|
|
|
1
|
-
#
|
|
1
|
+
# evals
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
This is the Javascript CLI for Umbrage Evals. It also supports TypeScript.
|
|
3
|
+
An npm package for Arize evals functionality.
|
|
6
4
|
|
|
7
5
|
## Installation
|
|
8
6
|
|
|
9
|
-
To install the package, run the following command:
|
|
10
|
-
|
|
11
7
|
```bash
|
|
12
|
-
npm install
|
|
8
|
+
npm install
|
|
13
9
|
```
|
|
14
10
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
- Node.js (or another JavaScript runtime environment like Bun)
|
|
18
|
-
|
|
19
|
-
## Environment
|
|
20
|
-
|
|
21
|
-
You will need to set the following environment variables:
|
|
22
|
-
|
|
23
|
-
```bash
|
|
24
|
-
UMBRAGE_EVALS_API_KEY=<your api key for the umbrage evals project>
|
|
25
|
-
OPENAI_API_KEY=<your openai api key>
|
|
26
|
-
```
|
|
11
|
+
The post-install script will automatically launch the CLI interface.
|
|
27
12
|
|
|
28
|
-
|
|
13
|
+
## Usage
|
|
29
14
|
|
|
30
|
-
|
|
15
|
+
Run the CLI manually:
|
|
31
16
|
|
|
32
17
|
```bash
|
|
33
|
-
|
|
34
|
-
export OPENAI_API_KEY=<your openai api key>
|
|
18
|
+
npm start
|
|
35
19
|
```
|
|
36
20
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
## Example usage
|
|
21
|
+
Or use the binary:
|
|
40
22
|
|
|
41
23
|
```bash
|
|
42
|
-
npx evals
|
|
43
|
-
|
|
44
|
-
npx evals@latest run-evals
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
## Example Prompt BaseModel
|
|
48
|
-
|
|
49
|
-
```typescript
|
|
50
|
-
export interface ModelSettings {
|
|
51
|
-
temperature: number;
|
|
52
|
-
[key: string]: any;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
export interface Prompt {
|
|
56
|
-
prompt_type: string;
|
|
57
|
-
prompt_text: string;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
export interface Config {
|
|
61
|
-
promptName: string;
|
|
62
|
-
modelName: string;
|
|
63
|
-
modelSettings: ModelSettings;
|
|
64
|
-
environment: string;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
class BaseModel {
|
|
68
|
-
promptName: string;
|
|
69
|
-
modelName: string;
|
|
70
|
-
modelSettings: ModelSettings;
|
|
71
|
-
environment: string;
|
|
72
|
-
|
|
73
|
-
constructor(config: Config) {
|
|
74
|
-
this.promptName = config.promptName;
|
|
75
|
-
this.modelName = config.modelName;
|
|
76
|
-
this.modelSettings = config.modelSettings;
|
|
77
|
-
this.environment = config.environment || 'development';
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* This method should be overridden in subclasses.
|
|
82
|
-
* It should return an object containing the response and the prompts array.
|
|
83
|
-
* @param userMessage The user message to be processed by the model, optional.
|
|
84
|
-
* @returns Promise<{ response: string, prompts: Prompt[] }>
|
|
85
|
-
*/
|
|
86
|
-
async callModel(userMessage: string = ''): Promise<{ response: string; prompts: Prompt[] }> {
|
|
87
|
-
throw new Error('callModel method needs to be implemented');
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
export default BaseModel;
|
|
24
|
+
npx evals
|
|
92
25
|
```
|
|
93
26
|
|
|
94
|
-
|
|
95
|
-
## LICENSE
|
|
96
|
-
|
|
97
|
-
Copyright (c) 2023 Umbrage Studios, LLC
|
|
98
|
-
|
|
99
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
100
|
-
|
|
101
|
-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
102
|
-
|
|
103
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/cli.js
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
const blessed = require('blessed');
|
|
4
|
+
const { exec } = require('child_process');
|
|
5
|
+
|
|
6
|
+
const LOGO_105 = `
|
|
7
|
+
{#FF008C-fg} {black-fg} @@@@{/black-fg} {/#FF008C-fg}
|
|
8
|
+
{#FF008C-fg} ** {black-fg} @@@@{/black-fg} {/#FF008C-fg}
|
|
9
|
+
{#FF008C-fg} ****** {black-fg} {/black-fg} {/#FF008C-fg}
|
|
10
|
+
{#FF008C-fg} ******** {black-fg}@@@@@@ @@@@ @@@ @@@ @@@@ @@@@@@@@@@@@@ @@@@@@{/black-fg} {/#FF008C-fg}
|
|
11
|
+
{#FF008C-fg} *********** {black-fg}@@@@@@@@@@@@@@@ @@@@@@@@@ @@@@ @@@@@@@@@@@@@@ @@@@@@@@@@@@{/black-fg} {/#FF008C-fg}
|
|
12
|
+
{#FF008C-fg} ****** ****** {black-fg}@@@@@ @@@@@@ @@@@@@@ @@@@ @@@@@@ @@@@@@ @@@@@{/black-fg} {/#FF008C-fg}
|
|
13
|
+
{#FF008C-fg} ****** ****** {black-fg}@@@@@ @@@@@ @@@@@ @@@@ @@@@@ @@@@ @@@@{/black-fg}{/#FF008C-fg}
|
|
14
|
+
{#FF008C-fg} ****** ****** {black-fg}@@@@ @@@@ @@@@ @@@@ @@@@@@ @@@@@@@@@@@@@@@@{/black-fg}{/#FF008C-fg}
|
|
15
|
+
{#FF008C-fg} ****** ****** {black-fg}@@@@ @@@@@ @@@@ @@@@ @@@@@ @@@@@@@@@@@@@@@@{/black-fg}{/#FF008C-fg}
|
|
16
|
+
{#FF008C-fg} ****** ****** {black-fg}@@@@@ @@@@@@ @@@@ @@@@ @@@@@ @@@@@ {/black-fg}{/#FF008C-fg}
|
|
17
|
+
{#FF008C-fg} ***** ***** {black-fg}@@@@@@@@@@@@@@@@ @@@@ @@@@ @@@@@@@@@@@@@@ @@@@@@@@@@@@@@{/black-fg} {/#FF008C-fg}
|
|
18
|
+
{#FF008C-fg} * **** * {black-fg}@@@@@@@@@ @@@@ @@@@ @@@@ @@@@@@@@@@@@@@ @@@@@@@@@@@{/black-fg} {/#FF008C-fg}
|
|
19
|
+
{#FF008C-fg} ****** {/#FF008C-fg}
|
|
20
|
+
{#FF008C-fg} ***** {/#FF008C-fg}
|
|
21
|
+
`;
|
|
22
|
+
|
|
23
|
+
const LOGO_75 = `
|
|
24
|
+
{#FF008C-fg} {black-fg}@@@@{/black-fg} {/#FF008C-fg}
|
|
25
|
+
{#FF008C-fg} ***** {black-fg}@@@@{/black-fg} {/#FF008C-fg}
|
|
26
|
+
{#FF008C-fg} ******* {black-fg}@@@@@@@@@@ @@@@@@@ @@@@ @@@@@@@@@@ @@@@@@@@@{/black-fg} {/#FF008C-fg}
|
|
27
|
+
{#FF008C-fg} ***** ***** {black-fg}@@@@@ @@@@@ @@@@@@@ @@@@ @@@@@@@@@ @@@@@ @@@@{/black-fg}{/#FF008C-fg}
|
|
28
|
+
{#FF008C-fg} ***** ***** {black-fg}@@@ @@@ @@@@ @@@@ @@@@ @@@@@@@@@@@@{/black-fg}{/#FF008C-fg}
|
|
29
|
+
{#FF008C-fg} ***** ***** {black-fg}@@@ @@@ @@@ @@@@ @@@@@ @@@@@@ {/black-fg}{/#FF008C-fg}
|
|
30
|
+
{#FF008C-fg} **** ***** {black-fg}@@@@@@@@@@@@ @@@ @@@@ @@@@@@@@@@ @@@@@@@@@@@{/black-fg}{/#FF008C-fg}
|
|
31
|
+
{#FF008C-fg} * *** * {black-fg}@@@@@@@@@@ @@@ @@@@ @@@@@@@@@@ @@@@@@@@{/black-fg} {/#FF008C-fg}
|
|
32
|
+
{#FF008C-fg} ***** {/#FF008C-fg}
|
|
33
|
+
`
|
|
34
|
+
|
|
35
|
+
const LOGO_51 = `
|
|
36
|
+
{#FF008C-fg} * {black-fg}@@@{/black-fg} {/#FF008C-fg}
|
|
37
|
+
{#FF008C-fg} **** {/#FF008C-fg}
|
|
38
|
+
{#FF008C-fg} ****** {black-fg} @@@@@@@@ @@@@@ @@ @@@@@@@ @@@@@@ {/black-fg}{/#FF008C-fg}
|
|
39
|
+
{#FF008C-fg} **** **** {black-fg}@@ @@@ @@ @@ @@@ @@@@ @@@{/black-fg}{/#FF008C-fg}
|
|
40
|
+
{#FF008C-fg}**** *** {black-fg}@@ @@@ @@ @@ @@@ @@ {/black-fg}{/#FF008C-fg}
|
|
41
|
+
{#FF008C-fg}** * *** {black-fg}@@@@@@@@ @@ @@ @@@@@@@ @@@@@@@{/black-fg}{/#FF008C-fg}
|
|
42
|
+
{#FF008C-fg} *** {/#FF008C-fg}
|
|
43
|
+
`
|
|
44
|
+
|
|
45
|
+
const OPTIONS = [
|
|
46
|
+
{ name: '1. Getting started with AI? Learn about evals.', value: 'option1' },
|
|
47
|
+
{ name: '2. Arize is now installed! Learn how to integrate it into your app.', value: 'option2' },
|
|
48
|
+
{ name: '3. I want to see what Arize does', value: 'option3' },
|
|
49
|
+
{ name: '4. I want to speak to an Arize expert', value: 'option4' }
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
function detectColorMode() {
|
|
53
|
+
// COLORFGBG format is typically "foreground;background" (e.g., "15;0")
|
|
54
|
+
const colorfgbg = process.env.COLORFGBG;
|
|
55
|
+
if (!colorfgbg) {
|
|
56
|
+
// Default to dark mode if COLORFGBG is not set
|
|
57
|
+
return 'dark';
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const parts = colorfgbg.split(';');
|
|
61
|
+
if (parts.length < 2) {
|
|
62
|
+
return 'dark';
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const bgColor = parseInt(parts[parts.length - 1], 10);
|
|
66
|
+
|
|
67
|
+
// Background colors: 0 = black (dark), 7 = white (light), 15 = bright white (light)
|
|
68
|
+
// Also consider 8-15 as potentially light colors
|
|
69
|
+
if (bgColor === 0 || (bgColor >= 1 && bgColor <= 6)) {
|
|
70
|
+
return 'dark';
|
|
71
|
+
} else {
|
|
72
|
+
return 'light';
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function openUrlInBrowser(url) {
|
|
77
|
+
console.log(`\nOpening ${url} in your browser...\n`);
|
|
78
|
+
|
|
79
|
+
// Open URL in browser (works on macOS, Linux, and Windows)
|
|
80
|
+
const command = process.platform === 'win32'
|
|
81
|
+
? `start ${url}`
|
|
82
|
+
: process.platform === 'darwin'
|
|
83
|
+
? `open "${url}"`
|
|
84
|
+
: `xdg-open "${url}"`;
|
|
85
|
+
|
|
86
|
+
exec(command, (error) => {
|
|
87
|
+
if (error) {
|
|
88
|
+
console.error(`Failed to open browser: ${error.message}`);
|
|
89
|
+
}
|
|
90
|
+
process.exit(0);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function showMenu() {
|
|
95
|
+
// Create a screen object
|
|
96
|
+
const screen = blessed.screen({
|
|
97
|
+
smartCSR: true,
|
|
98
|
+
title: 'Arize Evals'
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// Detect color mode from COLORFGBG
|
|
102
|
+
const colorMode = detectColorMode();
|
|
103
|
+
const isLightMode = colorMode === 'light';
|
|
104
|
+
|
|
105
|
+
// Select logo based on terminal width
|
|
106
|
+
const width = process.stdout.columns;
|
|
107
|
+
let selectedLogo;
|
|
108
|
+
let logoHeight;
|
|
109
|
+
let leftMargin;
|
|
110
|
+
if (width >= 105) {
|
|
111
|
+
selectedLogo = LOGO_105;
|
|
112
|
+
logoHeight = 16
|
|
113
|
+
leftMargin = 4
|
|
114
|
+
} else if (width >= 75) {
|
|
115
|
+
selectedLogo = LOGO_75;
|
|
116
|
+
logoHeight = 11
|
|
117
|
+
leftMargin = 2
|
|
118
|
+
} else {
|
|
119
|
+
selectedLogo = LOGO_51;
|
|
120
|
+
logoHeight = 9
|
|
121
|
+
leftMargin = 0
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Create a box for the ASCII art
|
|
125
|
+
const artBox = blessed.box({
|
|
126
|
+
top: 2,
|
|
127
|
+
left: 0,
|
|
128
|
+
width: '100%',
|
|
129
|
+
height: 'shrink',
|
|
130
|
+
content: selectedLogo,
|
|
131
|
+
tags: true,
|
|
132
|
+
style: {
|
|
133
|
+
bg: '#fff'
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
// Create a box for the menu title
|
|
138
|
+
const titleBox = blessed.box({
|
|
139
|
+
top: logoHeight + 2,
|
|
140
|
+
left: leftMargin,
|
|
141
|
+
width: 'shrink',
|
|
142
|
+
height: 1,
|
|
143
|
+
content: 'Select an option:',
|
|
144
|
+
tags: true,
|
|
145
|
+
style: {
|
|
146
|
+
fg: isLightMode ? 'black' : 'white',
|
|
147
|
+
bold: true
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
// Create a list for the menu options
|
|
152
|
+
const list = blessed.list({
|
|
153
|
+
top: logoHeight + 3,
|
|
154
|
+
left: leftMargin,
|
|
155
|
+
width: 80,
|
|
156
|
+
height: 8,
|
|
157
|
+
keys: true,
|
|
158
|
+
mouse: true,
|
|
159
|
+
vi: true,
|
|
160
|
+
items: OPTIONS.map(opt => opt.name),
|
|
161
|
+
style: {
|
|
162
|
+
selected: {
|
|
163
|
+
bg: 'blue',
|
|
164
|
+
fg: 'white',
|
|
165
|
+
bold: true
|
|
166
|
+
},
|
|
167
|
+
item: {
|
|
168
|
+
fg: isLightMode ? 'black' : 'white',
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
border: {
|
|
172
|
+
type: 'line'
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// Handle selection
|
|
177
|
+
list.on('select', (item) => {
|
|
178
|
+
const selectedIndex = list.selected;
|
|
179
|
+
const selectedOption = OPTIONS[selectedIndex];
|
|
180
|
+
handleOption(selectedOption.value, screen);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// Handle mouse clicks
|
|
184
|
+
list.on('click', (data) => {
|
|
185
|
+
const selectedIndex = list.selected;
|
|
186
|
+
const selectedOption = OPTIONS[selectedIndex];
|
|
187
|
+
handleOption(selectedOption.value, screen);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
// Quit on Escape, q, or Control-C
|
|
191
|
+
screen.key(['escape', 'q', 'C-c'], () => {
|
|
192
|
+
return screen.destroy();
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
// Append all the elements
|
|
196
|
+
screen.append(artBox);
|
|
197
|
+
screen.append(titleBox);
|
|
198
|
+
screen.append(list);
|
|
199
|
+
|
|
200
|
+
// Focus on the list so it can receive input
|
|
201
|
+
list.focus();
|
|
202
|
+
|
|
203
|
+
// Render the screen
|
|
204
|
+
screen.render();
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function handleOption(option, screen) {
|
|
208
|
+
screen.destroy();
|
|
209
|
+
console.clear();
|
|
210
|
+
|
|
211
|
+
if (option === 'option1') {
|
|
212
|
+
openUrlInBrowser('https://arize.com/docs/ax/evaluate/evaluators');
|
|
213
|
+
} else if (option === 'option2') {
|
|
214
|
+
openUrlInBrowser('https://arize.com/docs/ax/observe/tracing/setup');
|
|
215
|
+
} else if (option === 'option3') {
|
|
216
|
+
openUrlInBrowser('https://www.youtube.com/watch?v=QPMwpX8uzUI');
|
|
217
|
+
} else if (option === 'option4') {
|
|
218
|
+
openUrlInBrowser('?')
|
|
219
|
+
} else {
|
|
220
|
+
console.log(`\nYou selected: ${option}`);
|
|
221
|
+
console.log('(Implementation coming soon)\n');
|
|
222
|
+
process.exit(0);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Run the CLI
|
|
227
|
+
try {
|
|
228
|
+
showMenu();
|
|
229
|
+
} catch (error) {
|
|
230
|
+
console.error('Error:', error);
|
|
231
|
+
process.exit(1);
|
|
232
|
+
}
|
package/package.json
CHANGED
|
@@ -1,40 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "evals",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
"type": "module",
|
|
7
|
-
"version": "1.0.7",
|
|
8
|
-
"description": "Umbrage Evals CLI",
|
|
9
|
-
"bin": {
|
|
10
|
-
"evals": "./src/index.js"
|
|
11
|
-
},
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Arize evals package",
|
|
5
|
+
"main": "index.js",
|
|
12
6
|
"scripts": {
|
|
13
|
-
"
|
|
7
|
+
"postinstall": "node cli.js",
|
|
8
|
+
"start": "node cli.js"
|
|
14
9
|
},
|
|
15
|
-
"
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"rimraf": "^5.0.5",
|
|
19
|
-
"typescript": "^5.2.2"
|
|
10
|
+
"dependencies": {
|
|
11
|
+
"@arizeai/phoenix-otel": "*",
|
|
12
|
+
"blessed": "^0.1.81"
|
|
20
13
|
},
|
|
21
|
-
"
|
|
22
|
-
"
|
|
14
|
+
"bin": {
|
|
15
|
+
"evals": "./cli.js"
|
|
23
16
|
},
|
|
24
17
|
"keywords": [
|
|
25
|
-
"
|
|
26
|
-
"
|
|
27
|
-
"ai",
|
|
28
|
-
"cli"
|
|
29
|
-
],
|
|
30
|
-
"author": "umbrage.ai",
|
|
31
|
-
"license": "UNLICENSED",
|
|
32
|
-
"files": [
|
|
33
|
-
"dist/*.js",
|
|
34
|
-
"dist/*.d.ts"
|
|
18
|
+
"arize",
|
|
19
|
+
"evals"
|
|
35
20
|
],
|
|
36
|
-
"
|
|
37
|
-
|
|
38
|
-
"openai": "^4.24.1"
|
|
39
|
-
}
|
|
21
|
+
"author": "",
|
|
22
|
+
"license": "ISC"
|
|
40
23
|
}
|
|
24
|
+
|
package/LICENSE
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
Copyright (c) 2023 Umbrage Studios, LLC
|
|
2
|
-
|
|
3
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
-
|
|
5
|
-
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
-
|
|
7
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|