evals-lab 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/CHANGELOG.md +97 -0
  2. package/README.md +35 -14
  3. package/bin/evals-lab.js +8 -1
  4. package/bin/presets.js +234 -0
  5. package/lab/VERSION +1 -1
  6. package/lab/demo/manifest.json +2 -2
  7. package/lab/demo/pipelines/demo-1.json +105 -103
  8. package/lab/demo/pipelines/demo-2.json +105 -103
  9. package/lab/demo/pipelines/documents.json +54 -52
  10. package/lab/demo/pipelines/sentiment.json +30 -30
  11. package/lab/demo/pipelines/weather-alerts.json +35 -35
  12. package/lab/demo/pipelines/wordings.json +33 -36
  13. package/lab/evals-core.mjs +1053 -353
  14. package/lab/kinds/list.mjs +328 -224
  15. package/lab/metrics/builtin.mjs +24 -24
  16. package/lab/run-evals.js +34 -56
  17. package/lab/server.py +478 -206
  18. package/lab/web/dist/assets/gallery-ICJNZQIZ.js +5 -0
  19. package/lab/web/dist/assets/gallery-bG2vNzKT.css +1 -0
  20. package/lab/web/dist/assets/main-BOkQWswU.css +1 -0
  21. package/lab/web/dist/assets/main-rJDsPFWT.js +18 -0
  22. package/lab/web/dist/assets/tokens-C5ist-xG.js +64 -0
  23. package/lab/web/dist/assets/tokens-CuK6hUMK.css +1 -0
  24. package/lab/web/dist/gallery.html +4 -4
  25. package/lab/web/dist/index.html +4 -4
  26. package/package.json +1 -1
  27. package/lab/presets.json +0 -62
  28. package/lab/web/dist/assets/gallery-B_-TH0F-.css +0 -1
  29. package/lab/web/dist/assets/gallery-qrf0ZGXK.js +0 -5
  30. package/lab/web/dist/assets/main-D6xARg2A.js +0 -16
  31. package/lab/web/dist/assets/main-xVU6J7Gi.css +0 -1
  32. package/lab/web/dist/assets/tokens-C8dqXDHN.js +0 -62
  33. package/lab/web/dist/assets/tokens-tQ7zTPkd.css +0 -1
package/lab/demo/pipelines/demo-1.json CHANGED
@@ -1,107 +1,115 @@
1
1
  {
2
- "version": 18,
2
+ "version": 20,
3
3
  "id": "demo-1",
4
4
  "name": "Demo: images pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "Demo 1",
13
+ "name": "Demo: images source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "Demo 1",
17
- "name": "Demo: images source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
26
- {
27
- "type": "readReply",
28
- "out": {
29
- "kind": "list",
30
- "parse": "csv"
31
- }
32
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1",
33
+ "out": {
34
+ "kind": "list",
35
+ "parse": "csv"
36
+ }
37
+ },
38
+ {
39
+ "type": "filter",
40
+ "id": "filter-j1",
41
+ "request": "j1",
42
+ "filterSets": [
33
43
  {
34
- "type": "filters",
35
- "stage": "items",
36
- "filterSets": [
37
- {
38
- "id": "filters-j1",
39
- "set": null,
40
- "pin": null,
41
- "own": {
42
- "version": 3,
43
- "filters": [
44
- {
45
- "id": "tidy",
46
- "type": "rewrite-item",
47
- "match": {},
48
- "options": {
49
- "ignoreFailures": false,
50
- "repeat": 1
51
- },
52
- "tests": [],
53
- "op": {
54
- "kind": "tidy"
55
- }
56
- },
57
- {
58
- "id": "already there",
59
- "type": "drop-item",
60
- "match": {
61
- "duplicate": true
62
- },
63
- "options": {
64
- "ignoreFailures": false,
65
- "repeat": 1
66
- },
67
- "tests": []
68
- },
69
- {
70
- "id": "keep first",
71
- "type": "keep-first",
72
- "match": {},
73
- "options": {
74
- "ignoreFailures": false,
75
- "repeat": 1
76
- },
77
- "tests": [],
78
- "op": {
79
- "n": 12
80
- }
81
- },
82
- {
83
- "id": "case",
84
- "type": "rewrite-item",
85
- "match": {},
86
- "options": {
87
- "ignoreFailures": false,
88
- "repeat": 1
89
- },
90
- "tests": [],
91
- "op": {
92
- "kind": "case",
93
- "mode": "LOWER"
94
- }
95
- }
96
- ]
44
+ "id": "filters-j1",
45
+ "set": null,
46
+ "pin": null,
47
+ "own": {
48
+ "version": 4,
49
+ "filters": [
50
+ {
51
+ "id": "tidy",
52
+ "type": "trim",
53
+ "match": {},
54
+ "options": {
55
+ "ignoreFailures": false,
56
+ "repeat": 1
57
+ },
58
+ "tests": [],
59
+ "op": {
60
+ "bullets": true
61
+ }
62
+ },
63
+ {
64
+ "id": "already there",
65
+ "type": "remove",
66
+ "match": {
67
+ "duplicate": true
68
+ },
69
+ "options": {
70
+ "ignoreFailures": false,
71
+ "repeat": 1
72
+ },
73
+ "tests": [],
74
+ "op": {
75
+ "kind": "items"
76
+ }
77
+ },
78
+ {
79
+ "id": "keep first",
80
+ "type": "keep-first",
81
+ "match": {},
82
+ "options": {
83
+ "ignoreFailures": false,
84
+ "repeat": 1
85
+ },
86
+ "tests": [],
87
+ "op": {
88
+ "n": 12,
89
+ "unit": "items"
90
+ }
91
+ },
92
+ {
93
+ "id": "case",
94
+ "type": "change-case",
95
+ "match": {},
96
+ "options": {
97
+ "ignoreFailures": false,
98
+ "repeat": 1
99
+ },
100
+ "tests": [],
101
+ "op": {
102
+ "mode": "LOWER"
103
+ }
97
104
  }
98
- }
99
- ]
105
+ ]
106
+ }
100
107
  }
101
- ]
108
+ ],
109
+ "format": 0
102
110
  }
103
111
  ],
104
- "targets": [
112
+ "variants": [
105
113
  {
106
114
  "id": "s1",
107
115
  "name": "",
@@ -110,12 +118,7 @@
110
118
  "name": "Vision"
111
119
  },
112
120
  "profile": null,
113
- "steps": [
114
- {
115
- "type": "prompt",
116
- "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
117
- }
118
- ]
121
+ "overrides": {}
119
122
  },
120
123
  {
121
124
  "id": "s2",
@@ -125,12 +128,11 @@
125
128
  "name": "Vision"
126
129
  },
127
130
  "profile": null,
128
- "steps": [
129
- {
130
- "type": "prompt",
131
+ "overrides": {
132
+ "j1": {
131
133
  "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
132
134
  }
133
- ]
135
+ }
134
136
  }
135
137
  ],
136
138
  "evals": [
package/lab/demo/pipelines/demo-2.json CHANGED
@@ -1,107 +1,115 @@
1
1
  {
2
- "version": 18,
2
+ "version": 20,
3
3
  "id": "demo-2",
4
4
  "name": "Demo: images 2 pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "Demo 2",
13
+ "name": "Demo: images 2 source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "Demo 2",
17
- "name": "Demo: images 2 source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
26
- {
27
- "type": "readReply",
28
- "out": {
29
- "kind": "list",
30
- "parse": "csv"
31
- }
32
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1",
33
+ "out": {
34
+ "kind": "list",
35
+ "parse": "csv"
36
+ }
37
+ },
38
+ {
39
+ "type": "filter",
40
+ "id": "filter-j1",
41
+ "request": "j1",
42
+ "filterSets": [
33
43
  {
34
- "type": "filters",
35
- "stage": "items",
36
- "filterSets": [
37
- {
38
- "id": "filters-j1",
39
- "set": null,
40
- "pin": null,
41
- "own": {
42
- "version": 3,
43
- "filters": [
44
- {
45
- "id": "tidy",
46
- "type": "rewrite-item",
47
- "match": {},
48
- "options": {
49
- "ignoreFailures": false,
50
- "repeat": 1
51
- },
52
- "tests": [],
53
- "op": {
54
- "kind": "tidy"
55
- }
56
- },
57
- {
58
- "id": "already there",
59
- "type": "drop-item",
60
- "match": {
61
- "duplicate": true
62
- },
63
- "options": {
64
- "ignoreFailures": false,
65
- "repeat": 1
66
- },
67
- "tests": []
68
- },
69
- {
70
- "id": "keep first",
71
- "type": "keep-first",
72
- "match": {},
73
- "options": {
74
- "ignoreFailures": false,
75
- "repeat": 1
76
- },
77
- "tests": [],
78
- "op": {
79
- "n": 12
80
- }
81
- },
82
- {
83
- "id": "case",
84
- "type": "rewrite-item",
85
- "match": {},
86
- "options": {
87
- "ignoreFailures": false,
88
- "repeat": 1
89
- },
90
- "tests": [],
91
- "op": {
92
- "kind": "case",
93
- "mode": "LOWER"
94
- }
95
- }
96
- ]
44
+ "id": "filters-j1",
45
+ "set": null,
46
+ "pin": null,
47
+ "own": {
48
+ "version": 4,
49
+ "filters": [
50
+ {
51
+ "id": "tidy",
52
+ "type": "trim",
53
+ "match": {},
54
+ "options": {
55
+ "ignoreFailures": false,
56
+ "repeat": 1
57
+ },
58
+ "tests": [],
59
+ "op": {
60
+ "bullets": true
61
+ }
62
+ },
63
+ {
64
+ "id": "already there",
65
+ "type": "remove",
66
+ "match": {
67
+ "duplicate": true
68
+ },
69
+ "options": {
70
+ "ignoreFailures": false,
71
+ "repeat": 1
72
+ },
73
+ "tests": [],
74
+ "op": {
75
+ "kind": "items"
76
+ }
77
+ },
78
+ {
79
+ "id": "keep first",
80
+ "type": "keep-first",
81
+ "match": {},
82
+ "options": {
83
+ "ignoreFailures": false,
84
+ "repeat": 1
85
+ },
86
+ "tests": [],
87
+ "op": {
88
+ "n": 12,
89
+ "unit": "items"
90
+ }
91
+ },
92
+ {
93
+ "id": "case",
94
+ "type": "change-case",
95
+ "match": {},
96
+ "options": {
97
+ "ignoreFailures": false,
98
+ "repeat": 1
99
+ },
100
+ "tests": [],
101
+ "op": {
102
+ "mode": "LOWER"
103
+ }
97
104
  }
98
- }
99
- ]
105
+ ]
106
+ }
100
107
  }
101
- ]
108
+ ],
109
+ "format": 0
102
110
  }
103
111
  ],
104
- "targets": [
112
+ "variants": [
105
113
  {
106
114
  "id": "s1",
107
115
  "name": "",
@@ -110,12 +118,7 @@
110
118
  "name": "Vision"
111
119
  },
112
120
  "profile": null,
113
- "steps": [
114
- {
115
- "type": "prompt",
116
- "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
117
- }
118
- ]
121
+ "overrides": {}
119
122
  },
120
123
  {
121
124
  "id": "s2",
@@ -125,12 +128,11 @@
125
128
  "name": "Vision"
126
129
  },
127
130
  "profile": null,
128
- "steps": [
129
- {
130
- "type": "prompt",
131
+ "overrides": {
132
+ "j1": {
131
133
  "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
132
134
  }
133
- ]
135
+ }
134
136
  }
135
137
  ],
136
138
  "evals": [
package/lab/demo/pipelines/documents.json CHANGED
@@ -1,61 +1,68 @@
1
1
  {
2
- "version": 18,
2
+ "version": 20,
3
3
  "id": "documents",
4
4
  "name": "Demo: documents pipeline",
5
- "jobs": [
5
+ "steps": [
6
6
  {
7
- "type": "job",
7
+ "type": "getItems",
8
+ "id": "items-j1",
9
+ "content": {
10
+ "type": "source",
11
+ "ref": {
12
+ "id": "documents",
13
+ "name": "Demo: documents source"
14
+ },
15
+ "first": null,
16
+ "loops": 1
17
+ }
18
+ },
19
+ {
20
+ "type": "request",
8
21
  "id": "j1",
9
22
  "name": "",
10
- "steps": [
11
- {
12
- "type": "attachContent",
13
- "content": {
14
- "type": "source",
15
- "ref": {
16
- "id": "documents",
17
- "name": "Demo: documents source"
18
- },
19
- "first": null,
20
- "loops": 1
21
- }
22
- },
23
- {
24
- "type": "attachImage"
25
- },
23
+ "cell": {
24
+ "type": "prompt",
25
+ "prompt": "Read the attached document. Reply with JSON only: {\"valid\": true or false (false if you can't read it as a document), \"type\": \"invoice\" | \"letter\" | \"receipt\" | \"report\" | \"other\"}"
26
+ },
27
+ "image": true
28
+ },
29
+ {
30
+ "type": "readResponse",
31
+ "id": "read-j1",
32
+ "request": "j1"
33
+ },
34
+ {
35
+ "type": "filter",
36
+ "id": "filter-j1",
37
+ "request": "j1",
38
+ "filterSets": [
26
39
  {
27
- "type": "filters",
28
- "stage": "items",
29
- "filterSets": [
30
- {
31
- "id": "filters-j1",
32
- "set": null,
33
- "pin": null,
34
- "own": {
35
- "version": 3,
36
- "filters": [
37
- {
38
- "id": "unfence",
39
- "type": "rewrite-text",
40
- "match": {},
41
- "options": {
42
- "ignoreFailures": false,
43
- "repeat": 1
44
- },
45
- "tests": [],
46
- "op": {
47
- "kind": "fences"
48
- }
49
- }
50
- ]
40
+ "id": "filters-j1",
41
+ "set": null,
42
+ "pin": null,
43
+ "own": {
44
+ "version": 4,
45
+ "filters": [
46
+ {
47
+ "id": "unfence",
48
+ "type": "remove",
49
+ "match": {},
50
+ "options": {
51
+ "ignoreFailures": false,
52
+ "repeat": 1
53
+ },
54
+ "tests": [],
55
+ "op": {
56
+ "kind": "fences"
57
+ }
51
58
  }
52
- }
53
- ]
59
+ ]
60
+ }
54
61
  }
55
62
  ]
56
63
  }
57
64
  ],
58
- "targets": [
65
+ "variants": [
59
66
  {
60
67
  "id": "s1",
61
68
  "name": "",
@@ -64,12 +71,7 @@
64
71
  "name": "Model"
65
72
  },
66
73
  "profile": null,
67
- "steps": [
68
- {
69
- "type": "prompt",
70
- "prompt": "Read the attached document. Reply with JSON only: {\"valid\": true or false (false if you can't read it as a document), \"type\": \"invoice\" | \"letter\" | \"receipt\" | \"report\" | \"other\"}"
71
- }
72
- ]
74
+ "overrides": {}
73
75
  }
74
76
  ],
75
77
  "evals": [