ruby_llm-agent-evaluations 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +25 -0
  3. data/LICENSE +21 -0
  4. data/README.md +332 -0
  5. data/app/models/ruby_llm/evaluations/application_record.rb +32 -0
  6. data/app/models/ruby_llm/evaluations/case_result.rb +39 -0
  7. data/app/models/ruby_llm/evaluations/metric_result.rb +22 -0
  8. data/app/models/ruby_llm/evaluations/run.rb +55 -0
  9. data/config/routes.rb +5 -0
  10. data/db/migrate/20261011000000_create_agent_evaluation_history.rb +145 -0
  11. data/db/migrate/20261011000100_rename_case_criteria_to_minimum_scores.rb +7 -0
  12. data/docs/images/dashboard-case.png +0 -0
  13. data/docs/images/dashboard-history.png +0 -0
  14. data/docs/images/dashboard-mobile.png +0 -0
  15. data/docs/images/dashboard-regression.png +0 -0
  16. data/docs/images/dashboard-without-judge.png +0 -0
  17. data/docs/images/html-report.png +0 -0
  18. data/examples/api_replay_eval.rb +41 -0
  19. data/examples/context_bloat_eval.rb +77 -0
  20. data/examples/fixtures/large_return_policy.json +8006 -0
  21. data/examples/fixtures/order-42.json +8 -0
  22. data/examples/ruby_llm_eval.rb +26 -0
  23. data/examples/semantic_actions_eval.rb +43 -0
  24. data/examples/support_eval.rb +52 -0
  25. data/exe/ruby_llm-agent-evaluations +5 -0
  26. data/lib/ruby_llm/evaluations/action_group.rb +21 -0
  27. data/lib/ruby_llm/evaluations/actions.rb +30 -0
  28. data/lib/ruby_llm/evaluations/adapters.rb +40 -0
  29. data/lib/ruby_llm/evaluations/api_replay.rb +426 -0
  30. data/lib/ruby_llm/evaluations/cli.rb +209 -0
  31. data/lib/ruby_llm/evaluations/context.rb +19 -0
  32. data/lib/ruby_llm/evaluations/dashboard/dashboard.css +939 -0
  33. data/lib/ruby_llm/evaluations/dashboard/error.erb +1 -0
  34. data/lib/ruby_llm/evaluations/dashboard/index.erb +46 -0
  35. data/lib/ruby_llm/evaluations/dashboard/layout.erb +18 -0
  36. data/lib/ruby_llm/evaluations/dashboard/run.erb +54 -0
  37. data/lib/ruby_llm/evaluations/dashboard.rb +270 -0
  38. data/lib/ruby_llm/evaluations/data.rb +55 -0
  39. data/lib/ruby_llm/evaluations/engine.rb +18 -0
  40. data/lib/ruby_llm/evaluations/evaluators.rb +65 -0
  41. data/lib/ruby_llm/evaluations/judge_metric.rb +42 -0
  42. data/lib/ruby_llm/evaluations/report.rb +248 -0
  43. data/lib/ruby_llm/evaluations/response.rb +50 -0
  44. data/lib/ruby_llm/evaluations/runner.rb +139 -0
  45. data/lib/ruby_llm/evaluations/suite.rb +171 -0
  46. data/lib/ruby_llm/evaluations/version.rb +7 -0
  47. data/lib/ruby_llm/evaluations.rb +48 -0
  48. metadata +156 -0
metadata ADDED
@@ -0,0 +1,156 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ruby_llm-agent-evaluations
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.2.0
5
+ platform: ruby
6
+ authors:
7
+ - RubyLLM Evaluations contributors
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 2026-10-12 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: activerecord
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '7.1'
19
+ - - "<"
20
+ - !ruby/object:Gem::Version
21
+ version: '9'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ version: '7.1'
29
+ - - "<"
30
+ - !ruby/object:Gem::Version
31
+ version: '9'
32
+ - !ruby/object:Gem::Dependency
33
+ name: pg
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ">="
37
+ - !ruby/object:Gem::Version
38
+ version: '1.5'
39
+ - - "<"
40
+ - !ruby/object:Gem::Version
41
+ version: '2'
42
+ type: :runtime
43
+ prerelease: false
44
+ version_requirements: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '1.5'
49
+ - - "<"
50
+ - !ruby/object:Gem::Version
51
+ version: '2'
52
+ - !ruby/object:Gem::Dependency
53
+ name: webrick
54
+ requirement: !ruby/object:Gem::Requirement
55
+ requirements:
56
+ - - "~>"
57
+ - !ruby/object:Gem::Version
58
+ version: '1.8'
59
+ type: :runtime
60
+ prerelease: false
61
+ version_requirements: !ruby/object:Gem::Requirement
62
+ requirements:
63
+ - - "~>"
64
+ - !ruby/object:Gem::Version
65
+ version: '1.8'
66
+ - !ruby/object:Gem::Dependency
67
+ name: webmock
68
+ requirement: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - "~>"
71
+ - !ruby/object:Gem::Version
72
+ version: '3.26'
73
+ type: :runtime
74
+ prerelease: false
75
+ version_requirements: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - "~>"
78
+ - !ruby/object:Gem::Version
79
+ version: '3.26'
80
+ description: Evaluate any callable Ruby LLM agent with semantic, relevance, and action
81
+ checks, persist runs in PostgreSQL through ActiveRecord, and inspect regressions
82
+ in a local dashboard.
83
+ executables:
84
+ - ruby_llm-agent-evaluations
85
+ extensions: []
86
+ extra_rdoc_files: []
87
+ files:
88
+ - CHANGELOG.md
89
+ - LICENSE
90
+ - README.md
91
+ - app/models/ruby_llm/evaluations/application_record.rb
92
+ - app/models/ruby_llm/evaluations/case_result.rb
93
+ - app/models/ruby_llm/evaluations/metric_result.rb
94
+ - app/models/ruby_llm/evaluations/run.rb
95
+ - config/routes.rb
96
+ - db/migrate/20261011000000_create_agent_evaluation_history.rb
97
+ - db/migrate/20261011000100_rename_case_criteria_to_minimum_scores.rb
98
+ - docs/images/dashboard-case.png
99
+ - docs/images/dashboard-history.png
100
+ - docs/images/dashboard-mobile.png
101
+ - docs/images/dashboard-regression.png
102
+ - docs/images/dashboard-without-judge.png
103
+ - docs/images/html-report.png
104
+ - examples/api_replay_eval.rb
105
+ - examples/context_bloat_eval.rb
106
+ - examples/fixtures/large_return_policy.json
107
+ - examples/fixtures/order-42.json
108
+ - examples/ruby_llm_eval.rb
109
+ - examples/semantic_actions_eval.rb
110
+ - examples/support_eval.rb
111
+ - exe/ruby_llm-agent-evaluations
112
+ - lib/ruby_llm/evaluations.rb
113
+ - lib/ruby_llm/evaluations/action_group.rb
114
+ - lib/ruby_llm/evaluations/actions.rb
115
+ - lib/ruby_llm/evaluations/adapters.rb
116
+ - lib/ruby_llm/evaluations/api_replay.rb
117
+ - lib/ruby_llm/evaluations/cli.rb
118
+ - lib/ruby_llm/evaluations/context.rb
119
+ - lib/ruby_llm/evaluations/dashboard.rb
120
+ - lib/ruby_llm/evaluations/dashboard/dashboard.css
121
+ - lib/ruby_llm/evaluations/dashboard/error.erb
122
+ - lib/ruby_llm/evaluations/dashboard/index.erb
123
+ - lib/ruby_llm/evaluations/dashboard/layout.erb
124
+ - lib/ruby_llm/evaluations/dashboard/run.erb
125
+ - lib/ruby_llm/evaluations/data.rb
126
+ - lib/ruby_llm/evaluations/engine.rb
127
+ - lib/ruby_llm/evaluations/evaluators.rb
128
+ - lib/ruby_llm/evaluations/judge_metric.rb
129
+ - lib/ruby_llm/evaluations/report.rb
130
+ - lib/ruby_llm/evaluations/response.rb
131
+ - lib/ruby_llm/evaluations/runner.rb
132
+ - lib/ruby_llm/evaluations/suite.rb
133
+ - lib/ruby_llm/evaluations/version.rb
134
+ homepage: https://github.com/luisgsilva950/ruby_llm-evaluations
135
+ licenses:
136
+ - MIT
137
+ metadata: {}
138
+ rdoc_options: []
139
+ require_paths:
140
+ - lib
141
+ required_ruby_version: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '3.2'
146
+ required_rubygems_version: !ruby/object:Gem::Requirement
147
+ requirements:
148
+ - - ">="
149
+ - !ruby/object:Gem::Version
150
+ version: '0'
151
+ requirements: []
152
+ rubygems_version: 3.6.2
153
+ specification_version: 4
154
+ summary: Code-defined agent evaluations with persistent history, regression reports,
155
+ and a dashboard
156
+ test_files: []