ruby_llm-contract 1.3.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/lib/ruby_llm/contract/eval/recommender.rb +10 -9
- data/lib/ruby_llm/contract/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b3d6985e56b8e410f796207f23b52e910b925f1bf4263353da65ca3f540c9535
|
|
4
|
+
data.tar.gz: 4cb5f7471dc5d97f3d3bd78fe41110cb6a9f28139e90029872fa612ed4ef3e9e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 62e15223ae05cfa64fd64959eb62858ace305a6f24e23e2c3f466219681eae4a1cd4c08addaf9d7d309f554c4ead57e1051d9b4b1eb55bcea4542ce8df23d789
|
|
7
|
+
data.tar.gz: e4c07a0a8b4706cbc062ca713ab9c8a2e87906b2fa4126a3b6d10f72ef30625ce6c8c5ea0a29ac30fef0b5251a06fd451bd1462c45289a0ab1cc939cdc25b989
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.3.1 (2026-10-11)
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
|
|
7
|
+
- **`Eval::Recommender` priced a call N times too low on a `compare_models(runs: N)`
|
|
8
|
+
comparison.** With `runs > 1` each report is an `AggregatedReport`: its results
|
|
9
|
+
hold the cases of all N runs, while `total_cost` is the mean cost of one run. The
|
|
10
|
+
cost per call divided one run's cost by the cases of every run, so with `runs: 3`
|
|
11
|
+
a $0.002 call showed as $0.0007. That skewed `best`, the retry chain, the rationale
|
|
12
|
+
and `savings`, and a candidate measured over more runs looked cheaper than one
|
|
13
|
+
measured once. Cases are now counted per run. `Step.recommend` always compares
|
|
14
|
+
with one run, so only a `Recommender` built by hand on a multi-run comparison
|
|
15
|
+
was affected.
|
|
16
|
+
|
|
3
17
|
## 1.3.0 (2026-10-11)
|
|
4
18
|
|
|
5
19
|
### Changed - may turn a green CI red
|
|
@@ -40,15 +40,11 @@ module RubyLLM
|
|
|
40
40
|
report = @comparison.reports[label]
|
|
41
41
|
next nil unless report
|
|
42
42
|
|
|
43
|
-
evaluated_count = report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
|
|
44
|
-
cases_count = [evaluated_count, 1].max
|
|
45
|
-
cost_per_call = report.total_cost.to_f / cases_count
|
|
46
|
-
|
|
47
43
|
{
|
|
48
44
|
label: label,
|
|
49
45
|
config: config,
|
|
50
46
|
score: report.score,
|
|
51
|
-
cost_per_call: cost_per_call,
|
|
47
|
+
cost_per_call: cost_per_call(report),
|
|
52
48
|
latency: report.avg_latency_ms || Float::INFINITY,
|
|
53
49
|
pass_rate_ratio: report.pass_rate_ratio,
|
|
54
50
|
total_cost: report.total_cost,
|
|
@@ -117,15 +113,20 @@ module RubyLLM
|
|
|
117
113
|
current_report = @comparison.reports[current_label]
|
|
118
114
|
return {} if current_report.nil? || unknown_cost?(current_report)
|
|
119
115
|
|
|
120
|
-
|
|
121
|
-
current_cases = [current_evaluated, 1].max
|
|
122
|
-
current_cost = current_report.total_cost.to_f / current_cases
|
|
123
|
-
diff = current_cost - best[:cost_per_call]
|
|
116
|
+
diff = cost_per_call(current_report) - best[:cost_per_call]
|
|
124
117
|
return {} unless diff.positive?
|
|
125
118
|
|
|
126
119
|
{ per_call: diff.round(6), monthly_at: { 10_000 => (diff * 10_000).round(2) } }
|
|
127
120
|
end
|
|
128
121
|
|
|
122
|
+
# An AggregatedReport's results hold the cases of every run while its
|
|
123
|
+
# total_cost is the mean of one run, so cases are counted per run.
|
|
124
|
+
def cost_per_call(report)
|
|
125
|
+
runs = report.respond_to?(:runs) ? report.runs.length : 1
|
|
126
|
+
evaluated = report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
|
|
127
|
+
report.total_cost.to_f * runs / [evaluated, 1].max
|
|
128
|
+
end
|
|
129
|
+
|
|
129
130
|
# A subtotal over unpriced cases is not a cost to rank by. Zero stays
|
|
130
131
|
# "unknown pricing" too: offline runs report $0 without any call.
|
|
131
132
|
def cost_known?(scored_candidate)
|