ruby_llm-contract 0.4.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/.rubycritic.yml +8 -0
  3. data/.simplecov +22 -0
  4. data/CHANGELOG.md +19 -0
  5. data/Gemfile +2 -0
  6. data/Gemfile.lock +104 -2
  7. data/README.md +42 -2
  8. data/lib/ruby_llm/contract/concerns/context_helpers.rb +11 -10
  9. data/lib/ruby_llm/contract/concerns/deep_freeze.rb +13 -7
  10. data/lib/ruby_llm/contract/concerns/deep_symbolize.rb +15 -5
  11. data/lib/ruby_llm/contract/concerns/eval_host.rb +51 -7
  12. data/lib/ruby_llm/contract/contract/schema_validator/bound_rule.rb +85 -0
  13. data/lib/ruby_llm/contract/contract/schema_validator/enum_rule.rb +23 -0
  14. data/lib/ruby_llm/contract/contract/schema_validator/node.rb +70 -0
  15. data/lib/ruby_llm/contract/contract/schema_validator/object_rules.rb +66 -0
  16. data/lib/ruby_llm/contract/contract/schema_validator/scalar_rules.rb +22 -0
  17. data/lib/ruby_llm/contract/contract/schema_validator/schema_extractor.rb +23 -0
  18. data/lib/ruby_llm/contract/contract/schema_validator/type_rule.rb +30 -0
  19. data/lib/ruby_llm/contract/contract/schema_validator.rb +41 -266
  20. data/lib/ruby_llm/contract/contract/validator.rb +9 -0
  21. data/lib/ruby_llm/contract/eval/case_executor.rb +52 -0
  22. data/lib/ruby_llm/contract/eval/case_result_builder.rb +35 -0
  23. data/lib/ruby_llm/contract/eval/case_scorer.rb +66 -0
  24. data/lib/ruby_llm/contract/eval/evaluator/exact.rb +8 -6
  25. data/lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb +22 -10
  26. data/lib/ruby_llm/contract/eval/evaluator/regex.rb +11 -8
  27. data/lib/ruby_llm/contract/eval/expectation_evaluator.rb +26 -0
  28. data/lib/ruby_llm/contract/eval/prompt_diff.rb +39 -0
  29. data/lib/ruby_llm/contract/eval/prompt_diff_comparator.rb +116 -0
  30. data/lib/ruby_llm/contract/eval/prompt_diff_presenter.rb +99 -0
  31. data/lib/ruby_llm/contract/eval/prompt_diff_serializer.rb +23 -0
  32. data/lib/ruby_llm/contract/eval/report.rb +19 -191
  33. data/lib/ruby_llm/contract/eval/report_presenter.rb +65 -0
  34. data/lib/ruby_llm/contract/eval/report_stats.rb +65 -0
  35. data/lib/ruby_llm/contract/eval/report_storage.rb +107 -0
  36. data/lib/ruby_llm/contract/eval/runner.rb +30 -207
  37. data/lib/ruby_llm/contract/eval/step_expectation_applier.rb +67 -0
  38. data/lib/ruby_llm/contract/eval/step_result_normalizer.rb +39 -0
  39. data/lib/ruby_llm/contract/eval.rb +13 -0
  40. data/lib/ruby_llm/contract/pipeline/base.rb +10 -1
  41. data/lib/ruby_llm/contract/rspec/pass_eval.rb +84 -3
  42. data/lib/ruby_llm/contract/rspec.rb +5 -0
  43. data/lib/ruby_llm/contract/step/adapter_caller.rb +23 -0
  44. data/lib/ruby_llm/contract/step/base.rb +93 -38
  45. data/lib/ruby_llm/contract/step/dsl.rb +10 -0
  46. data/lib/ruby_llm/contract/step/input_validator.rb +34 -0
  47. data/lib/ruby_llm/contract/step/limit_checker.rb +11 -11
  48. data/lib/ruby_llm/contract/step/prompt_compiler.rb +33 -0
  49. data/lib/ruby_llm/contract/step/result.rb +3 -2
  50. data/lib/ruby_llm/contract/step/result_builder.rb +60 -0
  51. data/lib/ruby_llm/contract/step/retry_executor.rb +1 -0
  52. data/lib/ruby_llm/contract/step/runner.rb +46 -85
  53. data/lib/ruby_llm/contract/step/runner_config.rb +37 -0
  54. data/lib/ruby_llm/contract/step.rb +5 -0
  55. data/lib/ruby_llm/contract/version.rb +1 -1
  56. metadata +28 -1
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ruby_llm-contract
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.5
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justyna
@@ -60,6 +60,8 @@ extra_rdoc_files: []
60
60
  files:
61
61
  - ".rspec"
62
62
  - ".rubocop.yml"
63
+ - ".rubycritic.yml"
64
+ - ".simplecov"
63
65
  - CHANGELOG.md
64
66
  - Gemfile
65
67
  - Gemfile.lock
@@ -95,13 +97,23 @@ files:
95
97
  - lib/ruby_llm/contract/contract/invariant.rb
96
98
  - lib/ruby_llm/contract/contract/parser.rb
97
99
  - lib/ruby_llm/contract/contract/schema_validator.rb
100
+ - lib/ruby_llm/contract/contract/schema_validator/bound_rule.rb
101
+ - lib/ruby_llm/contract/contract/schema_validator/enum_rule.rb
102
+ - lib/ruby_llm/contract/contract/schema_validator/node.rb
103
+ - lib/ruby_llm/contract/contract/schema_validator/object_rules.rb
104
+ - lib/ruby_llm/contract/contract/schema_validator/scalar_rules.rb
105
+ - lib/ruby_llm/contract/contract/schema_validator/schema_extractor.rb
106
+ - lib/ruby_llm/contract/contract/schema_validator/type_rule.rb
98
107
  - lib/ruby_llm/contract/contract/validator.rb
99
108
  - lib/ruby_llm/contract/cost_calculator.rb
100
109
  - lib/ruby_llm/contract/dsl.rb
101
110
  - lib/ruby_llm/contract/errors.rb
102
111
  - lib/ruby_llm/contract/eval.rb
103
112
  - lib/ruby_llm/contract/eval/baseline_diff.rb
113
+ - lib/ruby_llm/contract/eval/case_executor.rb
104
114
  - lib/ruby_llm/contract/eval/case_result.rb
115
+ - lib/ruby_llm/contract/eval/case_result_builder.rb
116
+ - lib/ruby_llm/contract/eval/case_scorer.rb
105
117
  - lib/ruby_llm/contract/eval/contract_detail_builder.rb
106
118
  - lib/ruby_llm/contract/eval/dataset.rb
107
119
  - lib/ruby_llm/contract/eval/eval_definition.rb
@@ -111,10 +123,20 @@ files:
111
123
  - lib/ruby_llm/contract/eval/evaluator/json_includes.rb
112
124
  - lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb
113
125
  - lib/ruby_llm/contract/eval/evaluator/regex.rb
126
+ - lib/ruby_llm/contract/eval/expectation_evaluator.rb
114
127
  - lib/ruby_llm/contract/eval/model_comparison.rb
115
128
  - lib/ruby_llm/contract/eval/pipeline_result_adapter.rb
129
+ - lib/ruby_llm/contract/eval/prompt_diff.rb
130
+ - lib/ruby_llm/contract/eval/prompt_diff_comparator.rb
131
+ - lib/ruby_llm/contract/eval/prompt_diff_presenter.rb
132
+ - lib/ruby_llm/contract/eval/prompt_diff_serializer.rb
116
133
  - lib/ruby_llm/contract/eval/report.rb
134
+ - lib/ruby_llm/contract/eval/report_presenter.rb
135
+ - lib/ruby_llm/contract/eval/report_stats.rb
136
+ - lib/ruby_llm/contract/eval/report_storage.rb
117
137
  - lib/ruby_llm/contract/eval/runner.rb
138
+ - lib/ruby_llm/contract/eval/step_expectation_applier.rb
139
+ - lib/ruby_llm/contract/eval/step_result_normalizer.rb
118
140
  - lib/ruby_llm/contract/eval/trait_evaluator.rb
119
141
  - lib/ruby_llm/contract/minitest.rb
120
142
  - lib/ruby_llm/contract/pipeline.rb
@@ -139,13 +161,18 @@ files:
139
161
  - lib/ruby_llm/contract/rspec/pass_eval.rb
140
162
  - lib/ruby_llm/contract/rspec/satisfy_contract.rb
141
163
  - lib/ruby_llm/contract/step.rb
164
+ - lib/ruby_llm/contract/step/adapter_caller.rb
142
165
  - lib/ruby_llm/contract/step/base.rb
143
166
  - lib/ruby_llm/contract/step/dsl.rb
167
+ - lib/ruby_llm/contract/step/input_validator.rb
144
168
  - lib/ruby_llm/contract/step/limit_checker.rb
169
+ - lib/ruby_llm/contract/step/prompt_compiler.rb
145
170
  - lib/ruby_llm/contract/step/result.rb
171
+ - lib/ruby_llm/contract/step/result_builder.rb
146
172
  - lib/ruby_llm/contract/step/retry_executor.rb
147
173
  - lib/ruby_llm/contract/step/retry_policy.rb
148
174
  - lib/ruby_llm/contract/step/runner.rb
175
+ - lib/ruby_llm/contract/step/runner_config.rb
149
176
  - lib/ruby_llm/contract/step/trace.rb
150
177
  - lib/ruby_llm/contract/token_estimator.rb
151
178
  - lib/ruby_llm/contract/types.rb