llm-evals 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +26 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +764 -0
  5. data/exe/evals +9 -0
  6. data/exe/llm-evals +14 -0
  7. data/lib/llm/evals/adapter.rb +47 -0
  8. data/lib/llm/evals/case.rb +36 -0
  9. data/lib/llm/evals/cli/cases.rb +48 -0
  10. data/lib/llm/evals/cli/command.rb +163 -0
  11. data/lib/llm/evals/cli/compare.rb +40 -0
  12. data/lib/llm/evals/cli/formatting.rb +16 -0
  13. data/lib/llm/evals/cli/leaderboard.rb +183 -0
  14. data/lib/llm/evals/cli/model_colors.rb +23 -0
  15. data/lib/llm/evals/cli/model_command.rb +437 -0
  16. data/lib/llm/evals/cli/progress_bars.rb +147 -0
  17. data/lib/llm/evals/cli/render.rb +37 -0
  18. data/lib/llm/evals/cli/run.rb +69 -0
  19. data/lib/llm/evals/cli/standings.rb +114 -0
  20. data/lib/llm/evals/cli/verify.rb +24 -0
  21. data/lib/llm/evals/cli/version.rb +15 -0
  22. data/lib/llm/evals/cli.rb +43 -0
  23. data/lib/llm/evals/completion.rb +11 -0
  24. data/lib/llm/evals/config.rb +198 -0
  25. data/lib/llm/evals/configuration.rb +173 -0
  26. data/lib/llm/evals/contender.rb +31 -0
  27. data/lib/llm/evals/discovery.rb +95 -0
  28. data/lib/llm/evals/experiment.rb +158 -0
  29. data/lib/llm/evals/helper.rb +98 -0
  30. data/lib/llm/evals/launcher.rb +37 -0
  31. data/lib/llm/evals/metric.rb +26 -0
  32. data/lib/llm/evals/model.rb +55 -0
  33. data/lib/llm/evals/providers/base.rb +93 -0
  34. data/lib/llm/evals/providers/catalog.rb +32 -0
  35. data/lib/llm/evals/providers/open_router.rb +64 -0
  36. data/lib/llm/evals/providers/registry.rb +112 -0
  37. data/lib/llm/evals/providers/type_safe.rb +53 -0
  38. data/lib/llm/evals/render/reports/html.rb +266 -0
  39. data/lib/llm/evals/render/reports/json.rb +172 -0
  40. data/lib/llm/evals/render/reports/pipeline.rb +153 -0
  41. data/lib/llm/evals/render/reports/terminal.rb +46 -0
  42. data/lib/llm/evals/reply.rb +18 -0
  43. data/lib/llm/evals/run_result.rb +14 -0
  44. data/lib/llm/evals/runtime.rb +74 -0
  45. data/lib/llm/evals/schema_validator.rb +84 -0
  46. data/lib/llm/evals/score.rb +26 -0
  47. data/lib/llm/evals/scorer.rb +45 -0
  48. data/lib/llm/evals/templates/html_report.css +206 -0
  49. data/lib/llm/evals/templates/html_report.html.erb +184 -0
  50. data/lib/llm/evals/templates/html_report.js +48 -0
  51. data/lib/llm/evals/testing/echo_adapter.rb +29 -0
  52. data/lib/llm/evals/testing/fake_model.rb +18 -0
  53. data/lib/llm/evals/testing/fake_provider.rb +67 -0
  54. data/lib/llm/evals/tracker.rb +80 -0
  55. data/lib/llm/evals/trackers/braintrust.rb +91 -0
  56. data/lib/llm/evals/trackers/local.rb +78 -0
  57. data/lib/llm/evals/trackers/logfire.rb +143 -0
  58. data/lib/llm/evals/trackers/registry.rb +82 -0
  59. data/lib/llm/evals/usage.rb +33 -0
  60. data/lib/llm/evals/verify.rb +49 -0
  61. data/lib/llm/evals/version.rb +7 -0
  62. data/lib/llm/evals/warnings.rb +98 -0
  63. data/lib/llm/evals/weighted_score.rb +116 -0
  64. data/lib/llm/evals.rb +83 -0
  65. metadata +274 -0
metadata ADDED
@@ -0,0 +1,274 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: llm-evals
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.4.0
5
+ platform: ruby
6
+ authors:
7
+ - Konstantin Gredeskoul
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: dotenv
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '3.2'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '3.2'
26
+ - !ruby/object:Gem::Dependency
27
+ name: dry-cli
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '1.4'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '1.4'
40
+ - !ruby/object:Gem::Dependency
41
+ name: dry-cli-autocomplete
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0.6'
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '0.6'
54
+ - !ruby/object:Gem::Dependency
55
+ name: dry-cli-help
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '0.6'
61
+ type: :runtime
62
+ prerelease: false
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '0.6'
68
+ - !ruby/object:Gem::Dependency
69
+ name: dry-cli-ui
70
+ requirement: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '0.7'
75
+ type: :runtime
76
+ prerelease: false
77
+ version_requirements: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '0.7'
82
+ - !ruby/object:Gem::Dependency
83
+ name: zeitwerk
84
+ requirement: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '2.8'
89
+ type: :runtime
90
+ prerelease: false
91
+ version_requirements: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - "~>"
94
+ - !ruby/object:Gem::Version
95
+ version: '2.8'
96
+ - !ruby/object:Gem::Dependency
97
+ name: pastel
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '0.8'
103
+ type: :runtime
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '0.8'
110
+ - !ruby/object:Gem::Dependency
111
+ name: ruby_llm
112
+ requirement: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - "~>"
115
+ - !ruby/object:Gem::Version
116
+ version: '2.1'
117
+ type: :runtime
118
+ prerelease: false
119
+ version_requirements: !ruby/object:Gem::Requirement
120
+ requirements:
121
+ - - "~>"
122
+ - !ruby/object:Gem::Version
123
+ version: '2.1'
124
+ - !ruby/object:Gem::Dependency
125
+ name: tty-box
126
+ requirement: !ruby/object:Gem::Requirement
127
+ requirements:
128
+ - - "~>"
129
+ - !ruby/object:Gem::Version
130
+ version: '0.7'
131
+ type: :runtime
132
+ prerelease: false
133
+ version_requirements: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - "~>"
136
+ - !ruby/object:Gem::Version
137
+ version: '0.7'
138
+ - !ruby/object:Gem::Dependency
139
+ name: tty-table
140
+ requirement: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - "~>"
143
+ - !ruby/object:Gem::Version
144
+ version: '0.12'
145
+ type: :runtime
146
+ prerelease: false
147
+ version_requirements: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - "~>"
150
+ - !ruby/object:Gem::Version
151
+ version: '0.12'
152
+ - !ruby/object:Gem::Dependency
153
+ name: logger
154
+ requirement: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - "~>"
157
+ - !ruby/object:Gem::Version
158
+ version: '1.6'
159
+ type: :runtime
160
+ prerelease: false
161
+ version_requirements: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - "~>"
164
+ - !ruby/object:Gem::Version
165
+ version: '1.6'
166
+ description: |2-
167
+
168
+ Run multi-case evaluation suites against datasets using any model available
169
+ through OpenRouter or the direct Jev adapter. Write an adapter that calls
170
+ your application and a Scorer that measures the outcomes you care about.
171
+ Compare models, or track how prompt and parameter changes affect results.
172
+ Combine metrics such as accuracy, errors, latency, and cost into weighted
173
+ scores, then review rankings and case-level results in an HTML report.
174
+ Integrates with Braintrust Evals and several others. Paid integration is
175
+ not required.
176
+ email:
177
+ - kigster@gmail.com
178
+ executables:
179
+ - evals
180
+ - llm-evals
181
+ extensions: []
182
+ extra_rdoc_files: []
183
+ files:
184
+ - CHANGELOG.md
185
+ - LICENSE.txt
186
+ - README.md
187
+ - exe/evals
188
+ - exe/llm-evals
189
+ - lib/llm/evals.rb
190
+ - lib/llm/evals/adapter.rb
191
+ - lib/llm/evals/case.rb
192
+ - lib/llm/evals/cli.rb
193
+ - lib/llm/evals/cli/cases.rb
194
+ - lib/llm/evals/cli/command.rb
195
+ - lib/llm/evals/cli/compare.rb
196
+ - lib/llm/evals/cli/formatting.rb
197
+ - lib/llm/evals/cli/leaderboard.rb
198
+ - lib/llm/evals/cli/model_colors.rb
199
+ - lib/llm/evals/cli/model_command.rb
200
+ - lib/llm/evals/cli/progress_bars.rb
201
+ - lib/llm/evals/cli/render.rb
202
+ - lib/llm/evals/cli/run.rb
203
+ - lib/llm/evals/cli/standings.rb
204
+ - lib/llm/evals/cli/verify.rb
205
+ - lib/llm/evals/cli/version.rb
206
+ - lib/llm/evals/completion.rb
207
+ - lib/llm/evals/config.rb
208
+ - lib/llm/evals/configuration.rb
209
+ - lib/llm/evals/contender.rb
210
+ - lib/llm/evals/discovery.rb
211
+ - lib/llm/evals/experiment.rb
212
+ - lib/llm/evals/helper.rb
213
+ - lib/llm/evals/launcher.rb
214
+ - lib/llm/evals/metric.rb
215
+ - lib/llm/evals/model.rb
216
+ - lib/llm/evals/providers/base.rb
217
+ - lib/llm/evals/providers/catalog.rb
218
+ - lib/llm/evals/providers/open_router.rb
219
+ - lib/llm/evals/providers/registry.rb
220
+ - lib/llm/evals/providers/type_safe.rb
221
+ - lib/llm/evals/render/reports/html.rb
222
+ - lib/llm/evals/render/reports/json.rb
223
+ - lib/llm/evals/render/reports/pipeline.rb
224
+ - lib/llm/evals/render/reports/terminal.rb
225
+ - lib/llm/evals/reply.rb
226
+ - lib/llm/evals/run_result.rb
227
+ - lib/llm/evals/runtime.rb
228
+ - lib/llm/evals/schema_validator.rb
229
+ - lib/llm/evals/score.rb
230
+ - lib/llm/evals/scorer.rb
231
+ - lib/llm/evals/templates/html_report.css
232
+ - lib/llm/evals/templates/html_report.html.erb
233
+ - lib/llm/evals/templates/html_report.js
234
+ - lib/llm/evals/testing/echo_adapter.rb
235
+ - lib/llm/evals/testing/fake_model.rb
236
+ - lib/llm/evals/testing/fake_provider.rb
237
+ - lib/llm/evals/tracker.rb
238
+ - lib/llm/evals/trackers/braintrust.rb
239
+ - lib/llm/evals/trackers/local.rb
240
+ - lib/llm/evals/trackers/logfire.rb
241
+ - lib/llm/evals/trackers/registry.rb
242
+ - lib/llm/evals/usage.rb
243
+ - lib/llm/evals/verify.rb
244
+ - lib/llm/evals/version.rb
245
+ - lib/llm/evals/warnings.rb
246
+ - lib/llm/evals/weighted_score.rb
247
+ homepage: https://github.com/kigster/llm-evals
248
+ licenses:
249
+ - MIT
250
+ metadata:
251
+ allowed_push_host: https://rubygems.org
252
+ homepage_uri: https://github.com/kigster/llm-evals
253
+ source_code_uri: https://github.com/kigster/llm-evals
254
+ changelog_uri: https://github.com/kigster/llm-evals/blob/main/CHANGELOG.md
255
+ rubygems_mfa_required: 'true'
256
+ rdoc_options: []
257
+ require_paths:
258
+ - lib
259
+ required_ruby_version: !ruby/object:Gem::Requirement
260
+ requirements:
261
+ - - ">="
262
+ - !ruby/object:Gem::Version
263
+ version: 3.4.0
264
+ required_rubygems_version: !ruby/object:Gem::Requirement
265
+ requirements:
266
+ - - ">="
267
+ - !ruby/object:Gem::Version
268
+ version: '0'
269
+ requirements: []
270
+ rubygems_version: 4.0.23
271
+ specification_version: 4
272
+ summary: A CLI for comparing LLMs, prompt configurations, datasets with cases, and
273
+ with a custom evaluation suites.
274
+ test_files: []