evals-lab 0.15.0 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +35 -14
  3. package/bin/evals-lab.js +8 -1
  4. package/bin/presets.js +234 -0
  5. package/lab/VERSION +1 -1
  6. package/lab/demo/pipelines/demo-1.json +105 -106
  7. package/lab/demo/pipelines/demo-2.json +105 -106
  8. package/lab/demo/pipelines/documents.json +54 -51
  9. package/lab/demo/pipelines/sentiment.json +30 -30
  10. package/lab/demo/pipelines/weather-alerts.json +35 -35
  11. package/lab/demo/pipelines/wordings.json +33 -36
  12. package/lab/evals-core.mjs +815 -194
  13. package/lab/kinds/list.mjs +2 -2
  14. package/lab/metrics/builtin.mjs +24 -24
  15. package/lab/run-evals.js +34 -56
  16. package/lab/server.py +350 -180
  17. package/lab/web/dist/assets/gallery-ICJNZQIZ.js +5 -0
  18. package/lab/web/dist/assets/{gallery-eVJoaK-6.css → gallery-bG2vNzKT.css} +1 -1
  19. package/lab/web/dist/assets/main-Crw-d4cy.js +18 -0
  20. package/lab/web/dist/assets/main-Dxf5iQLA.css +1 -0
  21. package/lab/web/dist/assets/tokens-C5ist-xG.js +64 -0
  22. package/lab/web/dist/assets/tokens-CuK6hUMK.css +1 -0
  23. package/lab/web/dist/gallery.html +4 -4
  24. package/lab/web/dist/index.html +4 -4
  25. package/package.json +1 -1
  26. package/lab/presets.json +0 -62
  27. package/lab/web/dist/assets/gallery-Cy8oycht.js +0 -5
  28. package/lab/web/dist/assets/main-B-LGmhPu.css +0 -1
  29. package/lab/web/dist/assets/main-wqNvXpWp.js +0 -15
  30. package/lab/web/dist/assets/tokens-CWVzmtid.css +0 -1
  31. package/lab/web/dist/assets/tokens-mCS6IZGh.js +0 -64
package/lab/demo/pipelines/demo-1.json CHANGED
@@ -1,110 +1,115 @@
1
1
  {
2
- "version": 19,
2
+ "version": 20,
3
3
  "id": "demo-1",
4
4
  "name": "Demo: images pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "Demo 1",
13
+ "name": "Demo: images source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "Demo 1",
17
- "name": "Demo: images source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
26
- {
27
- "type": "readReply",
28
- "out": {
29
- "kind": "list",
30
- "parse": "csv"
31
- }
32
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1",
33
+ "out": {
34
+ "kind": "list",
35
+ "parse": "csv"
36
+ }
37
+ },
38
+ {
39
+ "type": "filter",
40
+ "id": "filter-j1",
41
+ "request": "j1",
42
+ "filterSets": [
33
43
  {
34
- "type": "filters",
35
- "filterSets": [
36
- {
37
- "id": "filters-j1",
38
- "set": null,
39
- "pin": null,
40
- "own": {
41
- "version": 4,
42
- "filters": [
43
- {
44
- "id": "tidy",
45
- "type": "trim",
46
- "match": {},
47
- "options": {
48
- "ignoreFailures": false,
49
- "repeat": 1
50
- },
51
- "tests": [],
52
- "op": {
53
- "bullets": true
54
- }
55
- },
56
- {
57
- "id": "already there",
58
- "type": "remove",
59
- "match": {
60
- "duplicate": true
61
- },
62
- "options": {
63
- "ignoreFailures": false,
64
- "repeat": 1
65
- },
66
- "tests": [],
67
- "op": {
68
- "kind": "items"
69
- }
70
- },
71
- {
72
- "id": "keep first",
73
- "type": "keep-first",
74
- "match": {},
75
- "options": {
76
- "ignoreFailures": false,
77
- "repeat": 1
78
- },
79
- "tests": [],
80
- "op": {
81
- "n": 12,
82
- "unit": "items"
83
- }
84
- },
85
- {
86
- "id": "case",
87
- "type": "change-case",
88
- "match": {},
89
- "options": {
90
- "ignoreFailures": false,
91
- "repeat": 1
92
- },
93
- "tests": [],
94
- "op": {
95
- "mode": "LOWER"
96
- }
97
- }
98
- ]
44
+ "id": "filters-j1",
45
+ "set": null,
46
+ "pin": null,
47
+ "own": {
48
+ "version": 4,
49
+ "filters": [
50
+ {
51
+ "id": "tidy",
52
+ "type": "trim",
53
+ "match": {},
54
+ "options": {
55
+ "ignoreFailures": false,
56
+ "repeat": 1
57
+ },
58
+ "tests": [],
59
+ "op": {
60
+ "bullets": true
61
+ }
62
+ },
63
+ {
64
+ "id": "already there",
65
+ "type": "remove",
66
+ "match": {
67
+ "duplicate": true
68
+ },
69
+ "options": {
70
+ "ignoreFailures": false,
71
+ "repeat": 1
72
+ },
73
+ "tests": [],
74
+ "op": {
75
+ "kind": "items"
76
+ }
77
+ },
78
+ {
79
+ "id": "keep first",
80
+ "type": "keep-first",
81
+ "match": {},
82
+ "options": {
83
+ "ignoreFailures": false,
84
+ "repeat": 1
85
+ },
86
+ "tests": [],
87
+ "op": {
88
+ "n": 12,
89
+ "unit": "items"
90
+ }
91
+ },
92
+ {
93
+ "id": "case",
94
+ "type": "change-case",
95
+ "match": {},
96
+ "options": {
97
+ "ignoreFailures": false,
98
+ "repeat": 1
99
+ },
100
+ "tests": [],
101
+ "op": {
102
+ "mode": "LOWER"
103
+ }
99
104
  }
100
- }
101
- ],
102
- "format": 0
105
+ ]
106
+ }
103
107
  }
104
- ]
108
+ ],
109
+ "format": 0
105
110
  }
106
111
  ],
107
- "targets": [
112
+ "variants": [
108
113
  {
109
114
  "id": "s1",
110
115
  "name": "",
@@ -113,12 +118,7 @@
113
118
  "name": "Vision"
114
119
  },
115
120
  "profile": null,
116
- "steps": [
117
- {
118
- "type": "prompt",
119
- "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
120
- }
121
- ]
121
+ "overrides": {}
122
122
  },
123
123
  {
124
124
  "id": "s2",
@@ -128,12 +128,11 @@
128
128
  "name": "Vision"
129
129
  },
130
130
  "profile": null,
131
- "steps": [
132
- {
133
- "type": "prompt",
131
+ "overrides": {
132
+ "j1": {
134
133
  "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
135
134
  }
136
- ]
135
+ }
137
136
  }
138
137
  ],
139
138
  "evals": [
package/lab/demo/pipelines/demo-2.json CHANGED
@@ -1,110 +1,115 @@
1
1
  {
2
- "version": 19,
2
+ "version": 20,
3
3
  "id": "demo-2",
4
4
  "name": "Demo: images 2 pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "Demo 2",
13
+ "name": "Demo: images 2 source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "Demo 2",
17
- "name": "Demo: images 2 source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
26
- {
27
- "type": "readReply",
28
- "out": {
29
- "kind": "list",
30
- "parse": "csv"
31
- }
32
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1",
33
+ "out": {
34
+ "kind": "list",
35
+ "parse": "csv"
36
+ }
37
+ },
38
+ {
39
+ "type": "filter",
40
+ "id": "filter-j1",
41
+ "request": "j1",
42
+ "filterSets": [
33
43
  {
34
- "type": "filters",
35
- "filterSets": [
36
- {
37
- "id": "filters-j1",
38
- "set": null,
39
- "pin": null,
40
- "own": {
41
- "version": 4,
42
- "filters": [
43
- {
44
- "id": "tidy",
45
- "type": "trim",
46
- "match": {},
47
- "options": {
48
- "ignoreFailures": false,
49
- "repeat": 1
50
- },
51
- "tests": [],
52
- "op": {
53
- "bullets": true
54
- }
55
- },
56
- {
57
- "id": "already there",
58
- "type": "remove",
59
- "match": {
60
- "duplicate": true
61
- },
62
- "options": {
63
- "ignoreFailures": false,
64
- "repeat": 1
65
- },
66
- "tests": [],
67
- "op": {
68
- "kind": "items"
69
- }
70
- },
71
- {
72
- "id": "keep first",
73
- "type": "keep-first",
74
- "match": {},
75
- "options": {
76
- "ignoreFailures": false,
77
- "repeat": 1
78
- },
79
- "tests": [],
80
- "op": {
81
- "n": 12,
82
- "unit": "items"
83
- }
84
- },
85
- {
86
- "id": "case",
87
- "type": "change-case",
88
- "match": {},
89
- "options": {
90
- "ignoreFailures": false,
91
- "repeat": 1
92
- },
93
- "tests": [],
94
- "op": {
95
- "mode": "LOWER"
96
- }
97
- }
98
- ]
44
+ "id": "filters-j1",
45
+ "set": null,
46
+ "pin": null,
47
+ "own": {
48
+ "version": 4,
49
+ "filters": [
50
+ {
51
+ "id": "tidy",
52
+ "type": "trim",
53
+ "match": {},
54
+ "options": {
55
+ "ignoreFailures": false,
56
+ "repeat": 1
57
+ },
58
+ "tests": [],
59
+ "op": {
60
+ "bullets": true
61
+ }
62
+ },
63
+ {
64
+ "id": "already there",
65
+ "type": "remove",
66
+ "match": {
67
+ "duplicate": true
68
+ },
69
+ "options": {
70
+ "ignoreFailures": false,
71
+ "repeat": 1
72
+ },
73
+ "tests": [],
74
+ "op": {
75
+ "kind": "items"
76
+ }
77
+ },
78
+ {
79
+ "id": "keep first",
80
+ "type": "keep-first",
81
+ "match": {},
82
+ "options": {
83
+ "ignoreFailures": false,
84
+ "repeat": 1
85
+ },
86
+ "tests": [],
87
+ "op": {
88
+ "n": 12,
89
+ "unit": "items"
90
+ }
91
+ },
92
+ {
93
+ "id": "case",
94
+ "type": "change-case",
95
+ "match": {},
96
+ "options": {
97
+ "ignoreFailures": false,
98
+ "repeat": 1
99
+ },
100
+ "tests": [],
101
+ "op": {
102
+ "mode": "LOWER"
103
+ }
99
104
  }
100
- }
101
- ],
102
- "format": 0
105
+ ]
106
+ }
103
107
  }
104
- ]
108
+ ],
109
+ "format": 0
105
110
  }
106
111
  ],
107
- "targets": [
112
+ "variants": [
108
113
  {
109
114
  "id": "s1",
110
115
  "name": "",
@@ -113,12 +118,7 @@
113
118
  "name": "Vision"
114
119
  },
115
120
  "profile": null,
116
- "steps": [
117
- {
118
- "type": "prompt",
119
- "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
120
- }
121
- ]
121
+ "overrides": {}
122
122
  },
123
123
  {
124
124
  "id": "s2",
@@ -128,12 +128,11 @@
128
128
  "name": "Vision"
129
129
  },
130
130
  "profile": null,
131
- "steps": [
132
- {
133
- "type": "prompt",
131
+ "overrides": {
132
+ "j1": {
134
133
  "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
135
134
  }
136
- ]
135
+ }
137
136
  }
138
137
  ],
139
138
  "evals": [
package/lab/demo/pipelines/documents.json CHANGED
@@ -1,60 +1,68 @@
1
1
  {
2
- "version": 19,
2
+ "version": 20,
3
3
  "id": "documents",
4
4
  "name": "Demo: documents pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "documents",
13
+ "name": "Demo: documents source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "documents",
17
- "name": "Demo: documents source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "Read the attached document. Reply with JSON only: {\"valid\": true or false (false if you can't read it as a document), \"type\": \"invoice\" | \"letter\" | \"receipt\" | \"report\" | \"other\"}"
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1"
33
+ },
34
+ {
35
+ "type": "filter",
36
+ "id": "filter-j1",
37
+ "request": "j1",
38
+ "filterSets": [
26
39
  {
27
- "type": "filters",
28
- "filterSets": [
29
- {
30
- "id": "filters-j1",
31
- "set": null,
32
- "pin": null,
33
- "own": {
34
- "version": 4,
35
- "filters": [
36
- {
37
- "id": "unfence",
38
- "type": "remove",
39
- "match": {},
40
- "options": {
41
- "ignoreFailures": false,
42
- "repeat": 1
43
- },
44
- "tests": [],
45
- "op": {
46
- "kind": "fences"
47
- }
48
- }
49
- ]
40
+ "id": "filters-j1",
41
+ "set": null,
42
+ "pin": null,
43
+ "own": {
44
+ "version": 4,
45
+ "filters": [
46
+ {
47
+ "id": "unfence",
48
+ "type": "remove",
49
+ "match": {},
50
+ "options": {
51
+ "ignoreFailures": false,
52
+ "repeat": 1
53
+ },
54
+ "tests": [],
55
+ "op": {
56
+ "kind": "fences"
57
+ }
50
58
  }
51
- }
52
- ]
59
+ ]
60
+ }
53
61
  }
54
62
  ]
55
63
  }
56
64
  ],
57
- "targets": [
65
+ "variants": [
58
66
  {
59
67
  "id": "s1",
60
68
  "name": "",
@@ -63,12 +71,7 @@
63
71
  "name": "Model"
64
72
  },
65
73
  "profile": null,
66
- "steps": [
67
- {
68
- "type": "prompt",
69
- "prompt": "Read the attached document. Reply with JSON only: {\"valid\": true or false (false if you can't read it as a document), \"type\": \"invoice\" | \"letter\" | \"receipt\" | \"report\" | \"other\"}"
70
- }
71
- ]
74
+ "overrides": {}
72
75
  }
73
76
  ],
74
77
  "evals": [