completion-kit 0.28.45 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/completion_kit/application.js +52 -23
- data/app/assets/stylesheets/completion_kit/application.css +56 -15
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
- data/app/controllers/completion_kit/runs_controller.rb +18 -6
- data/app/helpers/completion_kit/application_helper.rb +110 -52
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +1 -1
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +36 -18
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +2 -2
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
- data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
- data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
- data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/onboarding/show.html.erb +2 -2
- data/app/views/completion_kit/prompts/_form.html.erb +2 -1
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +21 -11
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
- data/app/views/completion_kit/runs/_row.html.erb +10 -8
- data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/edit.html.erb +1 -1
- data/app/views/completion_kit/runs/new.html.erb +1 -1
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +1 -1
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
- data/lib/completion_kit/errors.rb +2 -2
- data/lib/completion_kit/version.rb +1 -1
- metadata +10 -1
|
@@ -151,18 +151,22 @@ module CompletionKit
|
|
|
151
151
|
model[:name]
|
|
152
152
|
end
|
|
153
153
|
|
|
154
|
-
def ck_grouped_models(models, selected = nil)
|
|
154
|
+
def ck_grouped_models(models, selected = nil, rejected: false)
|
|
155
|
+
kept = nil
|
|
155
156
|
if selected.present? && models.none? { |m| m[:id] == selected }
|
|
156
|
-
retired = CompletionKit::Model.find_by(model_id: selected)
|
|
157
|
+
retired = CompletionKit::Model.find_by(model_id: selected) unless rejected
|
|
157
158
|
if retired
|
|
158
159
|
models = models + [{ id: retired.model_id, name: "#{retired.display_name || retired.model_id} (retired)", provider: retired.provider }]
|
|
160
|
+
else
|
|
161
|
+
kept = [["#{selected} (unavailable)", selected]]
|
|
159
162
|
end
|
|
160
163
|
end
|
|
161
164
|
|
|
162
165
|
groups = models.group_by { |m| ck_model_optgroup_label(m) }
|
|
163
166
|
ordered_keys = groups.keys.sort_by { |label| ck_model_optgroup_sort_key(label) }
|
|
164
167
|
grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [ck_model_option_label(m), m[:id]] }] }
|
|
165
|
-
grouped_options_for_select(grouped, selected)
|
|
168
|
+
options = grouped_options_for_select(grouped, selected)
|
|
169
|
+
kept ? options_for_select(kept, selected) + options : options
|
|
166
170
|
end
|
|
167
171
|
|
|
168
172
|
# Optgroup label for the model select — mirrors the provider models table:
|
|
@@ -170,15 +174,15 @@ module CompletionKit
|
|
|
170
174
|
# else is a single group.
|
|
171
175
|
def ck_model_optgroup_label(model)
|
|
172
176
|
case model[:provider]
|
|
173
|
-
when "openrouter" then "OpenRouter
|
|
174
|
-
when "openai" then "OpenAI
|
|
177
|
+
when "openrouter" then "OpenRouter · #{model[:id].to_s.split("/", 2).first.delete_prefix("~")}"
|
|
178
|
+
when "openai" then "OpenAI · #{ck_openai_model_family(model[:id])}"
|
|
175
179
|
else ck_provider_label(model[:provider])
|
|
176
180
|
end
|
|
177
181
|
end
|
|
178
182
|
|
|
179
183
|
def ck_model_optgroup_sort_key(label)
|
|
180
|
-
if label.start_with?("OpenAI
|
|
181
|
-
[0, OPENAI_MODEL_FAMILY_ORDER.index(label.delete_prefix("OpenAI
|
|
184
|
+
if label.start_with?("OpenAI · ")
|
|
185
|
+
[0, OPENAI_MODEL_FAMILY_ORDER.index(label.delete_prefix("OpenAI · ")), label]
|
|
182
186
|
elsif label.start_with?("OpenRouter")
|
|
183
187
|
[2, 0, label]
|
|
184
188
|
else
|
|
@@ -200,11 +204,52 @@ module CompletionKit
|
|
|
200
204
|
:low
|
|
201
205
|
end
|
|
202
206
|
|
|
203
|
-
def
|
|
207
|
+
def ck_percent_points(fraction)
|
|
208
|
+
(BigDecimal(fraction.to_s) * 100).round
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def ck_percent(fraction)
|
|
212
|
+
"#{ck_percent_points(fraction)}%"
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def ck_check_verdict(passed, score = nil)
|
|
216
|
+
verdict = passed ? "Pass" : "Fail"
|
|
217
|
+
score.nil? ? verdict : "#{verdict} #{ck_percent(score)}"
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def ck_check_average(summary)
|
|
221
|
+
ck_percent(summary[:avg_check_score] || summary[:pass_rate])
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def ck_check_average_label(name, summary)
|
|
225
|
+
pass_rate = "#{ck_percent(summary[:pass_rate])} pass rate"
|
|
226
|
+
return "#{name} #{pass_rate}" unless summary[:avg_check_score]
|
|
227
|
+
|
|
228
|
+
"#{name} average score #{ck_percent(summary[:avg_check_score])}, #{pass_rate}"
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def ck_check_score_change(left, right)
|
|
232
|
+
return nil if left.nil? || right.nil?
|
|
233
|
+
|
|
234
|
+
ck_percent_points(right) - ck_percent_points(left)
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def ck_delta_chip(delta, unit = "")
|
|
238
|
+
tone, arrow = if delta.positive? then %w[positive ▲]
|
|
239
|
+
elsif delta.negative? then %w[negative ▼]
|
|
240
|
+
else %w[zero =]
|
|
241
|
+
end
|
|
242
|
+
content_tag(:span, class: "ck-delta ck-delta--#{tone}") do
|
|
243
|
+
content_tag(:span, arrow, class: "ck-delta__arrow", "aria-hidden": "true") + "#{"+" if delta.positive?}#{delta}#{unit}"
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def ck_check_badge(passed, score = nil)
|
|
248
|
+
suffix = score.nil? ? "" : " #{ck_percent(score)}"
|
|
204
249
|
if passed == true
|
|
205
|
-
content_tag(:span, "Pass", class: ck_badge_classes(:high))
|
|
250
|
+
content_tag(:span, "Pass#{suffix}", class: ck_badge_classes(:high))
|
|
206
251
|
elsif passed == false
|
|
207
|
-
content_tag(:span, "Fail", class: ck_badge_classes(:low))
|
|
252
|
+
content_tag(:span, "Fail#{suffix}", class: ck_badge_classes(:low))
|
|
208
253
|
else
|
|
209
254
|
content_tag(:span, "Pending", class: ck_badge_classes(:pending))
|
|
210
255
|
end
|
|
@@ -228,58 +273,71 @@ module CompletionKit
|
|
|
228
273
|
{heading: heading, judge_model: judge ? judge_model.presence : nil}
|
|
229
274
|
end
|
|
230
275
|
|
|
231
|
-
CHECK_KIND_LABELS = {
|
|
232
|
-
"contains" => "Contains a phrase",
|
|
233
|
-
"not_contains" => "Does not contain a phrase",
|
|
234
|
-
"equals" => "Equals exactly",
|
|
235
|
-
"regex" => "Matches a pattern",
|
|
236
|
-
"valid_json" => "Is valid JSON",
|
|
237
|
-
"json_path_equals" => "A JSON field equals a value",
|
|
238
|
-
"length_bounds" => "Length is within a range"
|
|
239
|
-
}.freeze
|
|
240
|
-
|
|
241
|
-
CHECK_COMPARE_TO_LABELS = {
|
|
242
|
-
"constant" => "A value you type",
|
|
243
|
-
"expected" => "Each row's expected value"
|
|
244
|
-
}.freeze
|
|
245
|
-
|
|
246
|
-
CHECK_TARGET_LABELS = {
|
|
247
|
-
"response_text" => "The response text",
|
|
248
|
-
"input_data" => "The input row",
|
|
249
|
-
"json_path" => "A value from the response JSON"
|
|
250
|
-
}.freeze
|
|
251
|
-
|
|
252
|
-
CHECK_FIELD_LABELS = {
|
|
253
|
-
"value" => "Text to look for",
|
|
254
|
-
"pattern" => "Pattern",
|
|
255
|
-
"json_path" => "JSON path",
|
|
256
|
-
"expected" => "Expected value",
|
|
257
|
-
"expected_path" => "Field in the answer key",
|
|
258
|
-
"compare_to" => "Compare against",
|
|
259
|
-
"target_path" => "Path into the JSON",
|
|
260
|
-
"min" => "Shortest allowed",
|
|
261
|
-
"max" => "Longest allowed",
|
|
262
|
-
"case_sensitive" => "Case sensitive",
|
|
263
|
-
"multiline" => "Multiline",
|
|
264
|
-
"trim" => "Trim whitespace"
|
|
265
|
-
}.freeze
|
|
266
|
-
|
|
267
276
|
def ck_check_kind_label(kind)
|
|
268
|
-
|
|
277
|
+
CompletionKit::Checks::Labels.kind(kind)
|
|
269
278
|
end
|
|
270
279
|
|
|
271
280
|
def ck_check_target_label(target)
|
|
272
|
-
|
|
281
|
+
CompletionKit::Checks::Labels.target(target)
|
|
273
282
|
end
|
|
274
283
|
|
|
275
|
-
def ck_check_field_label(key)
|
|
276
|
-
|
|
284
|
+
def ck_check_field_label(key, kind = nil)
|
|
285
|
+
CompletionKit::Checks::Labels.field(key, kind)
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def ck_check_field_hint(key, kind = nil)
|
|
289
|
+
CompletionKit::Checks::Labels.hint(key, kind)
|
|
277
290
|
end
|
|
278
291
|
|
|
279
292
|
def ck_check_field_value(key, value)
|
|
280
|
-
|
|
293
|
+
CompletionKit::Checks::Labels.value(key, value)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def ck_check_input_value(value)
|
|
297
|
+
value.is_a?(Array) || value.is_a?(Hash) ? value.to_json : value
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def ck_check_expected_input_value(value)
|
|
301
|
+
return value if value.nil? || (value.is_a?(String) && CompletionKit::Checks::Registry.parse_json(value) == value)
|
|
302
|
+
|
|
303
|
+
value.to_json
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def ck_check_spec_fields(config)
|
|
307
|
+
shown = config.except("check_kind", "target").reject do |key, _value|
|
|
308
|
+
CompletionKit::Checks::Registry::FLAG_KEYS.include?(key) && !CompletionKit::Checks::Flag.on?(config, key)
|
|
309
|
+
end
|
|
310
|
+
CompletionKit::Checks::Labels.in_form_order(shown)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def ck_check_spec_value(key, value)
|
|
314
|
+
return "Yes" if CompletionKit::Checks::Registry::FLAG_KEYS.include?(key.to_s)
|
|
315
|
+
return ck_check_field_value(key, value) if CompletionKit::Checks::Labels.worded?(key)
|
|
316
|
+
|
|
317
|
+
shown = key.to_s == "expected" ? ck_check_expected_input_value(value) : ck_check_input_value(value)
|
|
318
|
+
content_tag(:code, shown.to_s, class: "ck-check-spec__code")
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
def ck_check_field_invalid?(metric, key)
|
|
322
|
+
metric.check_error_keys.include?(key)
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def ck_check_field_aria(metric, key)
|
|
326
|
+
ck_check_field_invalid?(metric, key) ? { "aria-invalid" => "true" } : {}
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
def ck_check_score_by_options
|
|
330
|
+
CompletionKit::Checks::Labels::SCORE_BY.to_a
|
|
331
|
+
end
|
|
281
332
|
|
|
282
|
-
|
|
333
|
+
def ck_check_form_data
|
|
334
|
+
labels = CompletionKit::Checks::Labels
|
|
335
|
+
{
|
|
336
|
+
fields: CompletionKit::Checks::Registry::CONFIG_KEYS,
|
|
337
|
+
expected: CompletionKit::Checks::Registry::EXPECTED_KEYS,
|
|
338
|
+
labels: labels::FIELDS_BY_KIND, defaultLabels: labels::FIELDS,
|
|
339
|
+
hints: labels::HINTS_BY_KIND, defaultHints: labels::HINTS
|
|
340
|
+
}
|
|
283
341
|
end
|
|
284
342
|
|
|
285
343
|
def ck_result_change_badge(change)
|
|
@@ -25,6 +25,7 @@ module CompletionKit
|
|
|
25
25
|
status: "succeeded",
|
|
26
26
|
passed: result.passed,
|
|
27
27
|
ai_score: nil,
|
|
28
|
+
check_score: result.score,
|
|
28
29
|
ai_feedback: result.detail,
|
|
29
30
|
error_provider: nil, error_class: nil, error_status: nil, error_message: nil
|
|
30
31
|
)
|
|
@@ -36,23 +37,43 @@ module CompletionKit
|
|
|
36
37
|
private
|
|
37
38
|
|
|
38
39
|
def evaluate(response, config)
|
|
39
|
-
|
|
40
|
+
kind = config["check_kind"]
|
|
41
|
+
target_value = resolve_target(response, config, kind)
|
|
40
42
|
if target_value.equal?(Checks::TargetResolver::UNRESOLVED)
|
|
41
|
-
return
|
|
43
|
+
return ungraded(kind, "could not resolve target")
|
|
42
44
|
end
|
|
43
45
|
|
|
44
|
-
kind = config["check_kind"]
|
|
45
46
|
if config["compare_to"] == "expected" && Checks::Registry.compares_value?(kind)
|
|
46
|
-
expected_value =
|
|
47
|
+
expected_value = resolve_expected(response, config, kind)
|
|
47
48
|
if expected_value.equal?(Checks::ExpectedResolver::UNRESOLVED)
|
|
48
|
-
return
|
|
49
|
+
return ungraded(kind, "no expected value for this row")
|
|
49
50
|
end
|
|
50
|
-
config = config.merge(
|
|
51
|
+
config = config.merge(Checks::Registry.expected_key(kind) => expected_value)
|
|
51
52
|
end
|
|
52
53
|
|
|
53
54
|
Checks::Registry.fetch(kind).call(target_value, config)
|
|
54
55
|
end
|
|
55
56
|
|
|
57
|
+
def ungraded(kind, detail)
|
|
58
|
+
Checks::Result.new(passed: false, detail: detail, score: Checks::Registry.scores?(kind) ? 0.0 : nil)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def resolve_target(response, config, kind)
|
|
62
|
+
if Checks::Registry.raw_target?(kind)
|
|
63
|
+
Checks::TargetResolver.call_value(response, config)
|
|
64
|
+
else
|
|
65
|
+
Checks::TargetResolver.call(response, config)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def resolve_expected(response, config, kind)
|
|
70
|
+
if Checks::Registry.raw_expected?(kind)
|
|
71
|
+
Checks::ExpectedResolver.call_value(response, config)
|
|
72
|
+
else
|
|
73
|
+
Checks::ExpectedResolver.call(response, config)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
56
77
|
def record_terminal_failure!(error)
|
|
57
78
|
response = Response.find_by(id: @response_id)
|
|
58
79
|
return unless response
|
|
@@ -30,6 +30,7 @@ module CompletionKit
|
|
|
30
30
|
validate :metric_type_immutable_once_in_use, on: :update
|
|
31
31
|
|
|
32
32
|
before_validation :generate_key
|
|
33
|
+
before_validation :default_check_target, if: :check?
|
|
33
34
|
before_validation :normalize_rubric_bands, if: :llm_judge?
|
|
34
35
|
before_validation :set_defaults, if: :llm_judge?
|
|
35
36
|
|
|
@@ -94,6 +95,10 @@ module CompletionKit
|
|
|
94
95
|
RunMetric.exists?(metric_id: id) || reviews.exists? || metric_versions.exists?
|
|
95
96
|
end
|
|
96
97
|
|
|
98
|
+
def check_error_keys
|
|
99
|
+
errors.objects.select { |error| error.attribute == :check_config }.flat_map { |error| Array(error.options[:check_keys]) }.uniq
|
|
100
|
+
end
|
|
101
|
+
|
|
97
102
|
def as_json(options = {})
|
|
98
103
|
base = {
|
|
99
104
|
id: id, name: name, key: key, metric_type: metric_type,
|
|
@@ -129,67 +134,137 @@ module CompletionKit
|
|
|
129
134
|
|
|
130
135
|
kind = config["check_kind"]
|
|
131
136
|
unless CompletionKit::Checks::Registry.kinds.include?(kind)
|
|
132
|
-
|
|
137
|
+
add_check_error("needs #{check_label("check_kind")} to be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}", "check_kind")
|
|
133
138
|
return
|
|
134
139
|
end
|
|
135
140
|
|
|
136
141
|
validate_check_target(config)
|
|
137
142
|
validate_check_compare_to(config, kind)
|
|
138
143
|
validate_check_required_keys(config, kind)
|
|
139
|
-
|
|
144
|
+
if refused_list_value?(config, kind)
|
|
145
|
+
add_check_error("needs #{check_label("value", kind)} to be a single value, since only list_overlap takes a list", "value")
|
|
146
|
+
else
|
|
147
|
+
validate_check_kind_rules(config, kind)
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def refused_list_value?(config, kind)
|
|
152
|
+
config["value"].is_a?(Array) && kind != "list_overlap" && will_save_change_to_check_config?
|
|
140
153
|
end
|
|
141
154
|
|
|
142
155
|
def validate_check_compare_to(config, kind)
|
|
143
156
|
return unless config.key?("compare_to")
|
|
144
157
|
|
|
145
158
|
unless %w[constant expected].include?(config["compare_to"])
|
|
146
|
-
|
|
159
|
+
add_check_error("needs #{check_label("compare_to")} to be constant or expected", "compare_to")
|
|
147
160
|
return
|
|
148
161
|
end
|
|
149
162
|
if config["compare_to"] == "expected" && !CompletionKit::Checks::Registry.compares_value?(kind)
|
|
150
|
-
|
|
163
|
+
add_check_error("can grade against \"Each row's expected value\" only with #{CompletionKit::Checks::Registry.comparable_kinds.join(", ")}", "compare_to")
|
|
151
164
|
end
|
|
152
165
|
end
|
|
153
166
|
|
|
154
167
|
def validate_check_target(config)
|
|
155
168
|
target = config["target"].presence || "response_text"
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
169
|
+
path_given = config["target_path"].present?
|
|
170
|
+
if !CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
|
|
171
|
+
add_check_error("needs #{check_label("target")} to be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}", "target")
|
|
172
|
+
elsif target == "json_path" && !path_given
|
|
173
|
+
add_check_error("needs #{check_label("target_path")} filled in to check a value from the response JSON", "target_path")
|
|
174
|
+
elsif target != "json_path" && path_given && will_save_change_to_check_config?
|
|
175
|
+
add_check_error("needs #{check_label("target")} to be \"#{CompletionKit::Checks::Labels.target("json_path")}\" when #{check_label("target_path")} is filled in", "target")
|
|
161
176
|
end
|
|
162
177
|
end
|
|
163
178
|
|
|
179
|
+
def default_check_target
|
|
180
|
+
config = check_config
|
|
181
|
+
return unless config.is_a?(Hash) && config["target"].blank? && config["target_path"].present? && will_save_change_to_check_config?
|
|
182
|
+
|
|
183
|
+
self.check_config = config.merge("target" => "json_path")
|
|
184
|
+
end
|
|
185
|
+
|
|
164
186
|
def validate_check_required_keys(config, kind)
|
|
165
187
|
required = CompletionKit::Checks::Registry.required_keys.fetch(kind)
|
|
166
|
-
required -=
|
|
188
|
+
required -= [CompletionKit::Checks::Registry.expected_key(kind)] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
|
|
167
189
|
required.each do |required_key|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
elsif config[required_key].to_s.strip.empty?
|
|
171
|
-
errors.add(:check_config, "#{required_key} is required")
|
|
172
|
-
end
|
|
190
|
+
missing = required_key == "expected" ? !config.key?("expected") : config[required_key].to_s.strip.empty?
|
|
191
|
+
add_check_error("needs #{check_label(required_key, kind)} filled in", required_key) if missing
|
|
173
192
|
end
|
|
174
193
|
end
|
|
175
194
|
|
|
176
195
|
def validate_check_kind_rules(config, kind)
|
|
177
196
|
case kind
|
|
178
197
|
when "regex"
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
when "length_bounds"
|
|
185
|
-
min = config["min"]
|
|
186
|
-
max = config["max"]
|
|
187
|
-
if min.nil? && max.nil?
|
|
188
|
-
errors.add(:check_config, "length_bounds requires at least one of min or max")
|
|
189
|
-
elsif min && max && min.to_i > max.to_i
|
|
190
|
-
errors.add(:check_config, "min must be less than or equal to max")
|
|
191
|
-
end
|
|
198
|
+
validate_check_pattern(config)
|
|
199
|
+
when "list_overlap"
|
|
200
|
+
validate_check_list_overlap(config)
|
|
201
|
+
when "numeric_equals"
|
|
202
|
+
validate_check_numeric_equals(config)
|
|
192
203
|
end
|
|
204
|
+
validate_check_bounds(config, kind) if CompletionKit::Checks::Registry.bounded?(kind)
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def validate_check_pattern(config)
|
|
208
|
+
Regexp.new(config["pattern"].to_s)
|
|
209
|
+
rescue RegexpError
|
|
210
|
+
add_check_error("needs #{check_label("pattern")} to be a valid regular expression", "pattern")
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def validate_check_bounds(config, kind)
|
|
214
|
+
min = config["min"].to_s.strip
|
|
215
|
+
max = config["max"].to_s.strip
|
|
216
|
+
min_label = check_label("min", kind)
|
|
217
|
+
max_label = check_label("max", kind)
|
|
218
|
+
if min.empty? && max.empty?
|
|
219
|
+
add_check_error("needs #{min_label} or #{max_label} filled in", "min", "max")
|
|
220
|
+
return
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
min_value = CompletionKit::Checks::NumericValue.parse(min)
|
|
224
|
+
max_value = CompletionKit::Checks::NumericValue.parse(max)
|
|
225
|
+
not_numbers = { "min" => !min.empty? && min_value.nil?, "max" => !max.empty? && max_value.nil? }.select { |_key, bad| bad }.keys
|
|
226
|
+
if not_numbers.any?
|
|
227
|
+
add_check_error("needs #{min_label} and #{max_label} to be numbers", *not_numbers)
|
|
228
|
+
elsif min_value && max_value && min_value > max_value
|
|
229
|
+
add_check_error("needs #{min_label} to be no more than #{max_label}", "min", "max")
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def validate_check_list_overlap(config)
|
|
234
|
+
measures = CompletionKit::Checks::ListOverlap::MEASURES
|
|
235
|
+
if config["score_by"].present? && !measures.include?(config["score_by"])
|
|
236
|
+
add_check_error("needs #{check_label("score_by")} to be one of #{measures.join(", ")}", "score_by")
|
|
237
|
+
end
|
|
238
|
+
return if config["min"].to_s.strip.empty?
|
|
239
|
+
|
|
240
|
+
threshold = CompletionKit::Checks::NumericValue.parse(config["min"])
|
|
241
|
+
if threshold.nil? || threshold.negative? || threshold > 1
|
|
242
|
+
add_check_error("needs #{check_label("min", "list_overlap")} to be a number between 0 and 1", "min")
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def validate_check_numeric_equals(config)
|
|
247
|
+
validate_check_tolerance(config)
|
|
248
|
+
return if config["compare_to"] == "expected" || config["value"].to_s.strip.empty?
|
|
249
|
+
|
|
250
|
+
add_check_error("needs #{check_label("value", "numeric_equals")} to be a number", "value") if CompletionKit::Checks::NumericValue.parse(config["value"]).nil?
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
def validate_check_tolerance(config)
|
|
254
|
+
return if config["tolerance"].to_s.strip.empty?
|
|
255
|
+
|
|
256
|
+
tolerance = CompletionKit::Checks::NumericEquals.tolerance_value(config["tolerance"])
|
|
257
|
+
if tolerance.nil? || tolerance.negative?
|
|
258
|
+
add_check_error("needs #{check_label("tolerance")} to be a number that is not negative, or a percentage like 2%", "tolerance")
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def add_check_error(message, *keys)
|
|
263
|
+
errors.add(:check_config, message, check_keys: keys)
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def check_label(key, kind = nil)
|
|
267
|
+
"\"#{CompletionKit::Checks::Labels.field(key, kind)}\""
|
|
193
268
|
end
|
|
194
269
|
|
|
195
270
|
def set_defaults
|
|
@@ -85,12 +85,14 @@ module CompletionKit
|
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
def publish!
|
|
88
|
+
config = publishable_check_config if check?
|
|
89
|
+
validate_check_config_on_metric!(config) if check?
|
|
88
90
|
MetricVersion.transaction do
|
|
89
91
|
self.class.where(metric_id: metric_id).where.not(id: id).update_all(current: false)
|
|
90
92
|
reload
|
|
91
93
|
update!(state: "published", current: true, published_at: published_at || Time.current)
|
|
92
94
|
if check?
|
|
93
|
-
metric.update_columns(metric_type: "check", check_config:
|
|
95
|
+
metric.update_columns(metric_type: "check", check_config: config)
|
|
94
96
|
else
|
|
95
97
|
metric.update_columns(
|
|
96
98
|
metric_type: "llm_judge",
|
|
@@ -123,6 +125,19 @@ module CompletionKit
|
|
|
123
125
|
|
|
124
126
|
private
|
|
125
127
|
|
|
128
|
+
def publishable_check_config
|
|
129
|
+
config = check_config.to_h.stringify_keys
|
|
130
|
+
return config if config["target_path"].blank?
|
|
131
|
+
return config.merge("target" => "json_path") if config["target"].blank?
|
|
132
|
+
|
|
133
|
+
config["target"] == "json_path" ? config : config.except("target_path")
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def validate_check_config_on_metric!(config)
|
|
137
|
+
metric.assign_attributes(metric_type: "check", check_config: config)
|
|
138
|
+
raise ActiveRecord::RecordInvalid, metric unless metric.valid?
|
|
139
|
+
end
|
|
140
|
+
|
|
126
141
|
def check_change_summary_against(previous)
|
|
127
142
|
return nil if check_config == previous.check_config
|
|
128
143
|
|
|
@@ -10,6 +10,7 @@ module CompletionKit
|
|
|
10
10
|
validates :metric_name, presence: true
|
|
11
11
|
validates :metric_version, presence: true
|
|
12
12
|
validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
|
|
13
|
+
validates :check_score, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
|
|
13
14
|
|
|
14
15
|
before_validation :set_default_status
|
|
15
16
|
|
|
@@ -36,6 +37,7 @@ module CompletionKit
|
|
|
36
37
|
id: id, response_id: response_id, metric_id: metric_id,
|
|
37
38
|
metric_version_id: metric_version_id,
|
|
38
39
|
metric_name: metric_name, ai_score: ai_score, passed: passed,
|
|
40
|
+
check_score: check_score&.to_f,
|
|
39
41
|
ai_feedback: ai_feedback, status: status, attempts: attempts,
|
|
40
42
|
error: error_payload
|
|
41
43
|
}
|
|
@@ -10,7 +10,7 @@ module CompletionKit
|
|
|
10
10
|
status: "pending",
|
|
11
11
|
attempts: 0,
|
|
12
12
|
error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
|
|
13
|
-
ai_score: nil, passed: nil, ai_feedback: nil
|
|
13
|
+
ai_score: nil, passed: nil, check_score: nil, ai_feedback: nil
|
|
14
14
|
}.freeze
|
|
15
15
|
|
|
16
16
|
belongs_to :prompt, optional: true
|
|
@@ -79,7 +79,9 @@ module CompletionKit
|
|
|
79
79
|
Arel.sql("COUNT(ai_score)"),
|
|
80
80
|
Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
|
|
81
81
|
Arel.sql("COUNT(passed)"),
|
|
82
|
-
Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
|
|
82
|
+
Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)"),
|
|
83
|
+
Arel.sql("AVG(check_score)"),
|
|
84
|
+
Arel.sql("COUNT(check_score)")
|
|
83
85
|
)
|
|
84
86
|
metrics_by_run = metric_rows.group_by(&:first)
|
|
85
87
|
|
|
@@ -90,12 +92,12 @@ module CompletionKit
|
|
|
90
92
|
run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
|
|
91
93
|
run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil
|
|
92
94
|
|
|
93
|
-
run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
|
|
95
|
+
run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed, fraction_avg, fraction_count)|
|
|
94
96
|
if scored.to_i > 0
|
|
95
97
|
{name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
|
|
96
98
|
elsif resolved.to_i > 0
|
|
97
|
-
|
|
98
|
-
|
|
99
|
+
check_summary(name, passed.to_i, resolved.to_i,
|
|
100
|
+
fraction_count.to_i > 0 ? fraction_avg.to_f : nil)
|
|
99
101
|
end
|
|
100
102
|
end
|
|
101
103
|
end
|
|
@@ -109,6 +111,14 @@ module CompletionKit
|
|
|
109
111
|
CompletionKit.config.medium_quality_threshold.to_f
|
|
110
112
|
end
|
|
111
113
|
|
|
114
|
+
def self.check_summary(name, passed, resolved, fraction_avg)
|
|
115
|
+
summary = { name: name, kind: "check", pass_rate: (passed.to_f / resolved).round(2),
|
|
116
|
+
count: resolved, low_count: resolved - passed }
|
|
117
|
+
return summary if fraction_avg.nil?
|
|
118
|
+
|
|
119
|
+
summary.merge(avg_check_score: fraction_avg.round(3))
|
|
120
|
+
end
|
|
121
|
+
|
|
112
122
|
TOP_SCORE = 5.0
|
|
113
123
|
CEILING_MEAN = 4.8
|
|
114
124
|
CEILING_TOP_SHARE = 0.9
|
|
@@ -271,9 +281,9 @@ module CompletionKit
|
|
|
271
281
|
resolved = reviews.reject { |r| r.passed.nil? }
|
|
272
282
|
next if resolved.empty?
|
|
273
283
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
284
|
+
fractions = reviews.filter_map { |r| r.check_score&.to_f }
|
|
285
|
+
self.class.check_summary(name, resolved.count { |r| r.passed == true }, resolved.length,
|
|
286
|
+
fractions.any? ? fractions.sum / fractions.length : nil)
|
|
277
287
|
end
|
|
278
288
|
end
|
|
279
289
|
end
|
|
@@ -333,13 +343,10 @@ module CompletionKit
|
|
|
333
343
|
unless judge_only_input_data_checks? || (dataset && dataset.headers.include?(column))
|
|
334
344
|
return fail_with_summary!("Dataset has no \"#{column}\" column")
|
|
335
345
|
end
|
|
336
|
-
else
|
|
337
|
-
generation_error = generation_client_error
|
|
338
|
-
return fail_with_summary!(generation_error) if generation_error
|
|
339
346
|
end
|
|
340
347
|
|
|
341
|
-
|
|
342
|
-
return fail_with_summary!(
|
|
348
|
+
model_error = model_start_error
|
|
349
|
+
return fail_with_summary!(model_error) if model_error
|
|
343
350
|
|
|
344
351
|
begin
|
|
345
352
|
transaction do
|
|
@@ -449,6 +456,7 @@ module CompletionKit
|
|
|
449
456
|
metric_version_id: nil,
|
|
450
457
|
ai_score: nil,
|
|
451
458
|
passed: nil,
|
|
459
|
+
check_score: nil,
|
|
452
460
|
ai_feedback: nil,
|
|
453
461
|
error_provider: nil,
|
|
454
462
|
error_class: nil,
|
|
@@ -490,7 +498,8 @@ module CompletionKit
|
|
|
490
498
|
end
|
|
491
499
|
|
|
492
500
|
def rerun!
|
|
493
|
-
|
|
501
|
+
ensure_models_ready!
|
|
502
|
+
|
|
494
503
|
Run.create!(
|
|
495
504
|
prompt_id: prompt_id,
|
|
496
505
|
dataset_id: dataset_id,
|
|
@@ -712,11 +721,20 @@ module CompletionKit
|
|
|
712
721
|
%(No configured provider serves judge model "#{judge_model}")
|
|
713
722
|
end
|
|
714
723
|
|
|
715
|
-
def
|
|
716
|
-
|
|
724
|
+
def model_start_error
|
|
725
|
+
(generation_client_error unless judge_only?) || judge_model_error
|
|
726
|
+
end
|
|
727
|
+
|
|
728
|
+
def ensure_models_ready!
|
|
729
|
+
unless judge_only?
|
|
730
|
+
raise UnknownModelError, prompt.llm_model unless ApiConfig.provider_for_model(prompt.llm_model)
|
|
731
|
+
|
|
732
|
+
client_error = generation_client_error
|
|
733
|
+
raise ConfigurationError, client_error if client_error
|
|
734
|
+
end
|
|
717
735
|
|
|
718
736
|
judge_error = judge_model_error
|
|
719
|
-
raise
|
|
737
|
+
raise UnknownModelError.new(judge_model, judge_error) if judge_error
|
|
720
738
|
end
|
|
721
739
|
|
|
722
740
|
def judge_model_newly_selected?
|
|
@@ -775,7 +793,7 @@ module CompletionKit
|
|
|
775
793
|
|
|
776
794
|
if prompt.present?
|
|
777
795
|
count = Run.where(prompt_id: prompt_id).count + 1
|
|
778
|
-
self.name = "#{prompt.name}
|
|
796
|
+
self.name = "#{prompt.name} · v#{prompt.version_number} ##{count}"
|
|
779
797
|
elsif dataset.present?
|
|
780
798
|
count = Run.where(prompt_id: nil, dataset_id: dataset.id).count + 1
|
|
781
799
|
self.name = "#{dataset.name} scoring ##{count}"
|
|
@@ -2,15 +2,9 @@ module CompletionKit
|
|
|
2
2
|
module Checks
|
|
3
3
|
class Contains
|
|
4
4
|
def call(target, config)
|
|
5
|
-
value = config
|
|
6
|
-
haystack = target.to_s
|
|
7
|
-
present = if config["case_sensitive"] == true
|
|
8
|
-
haystack.include?(value)
|
|
9
|
-
else
|
|
10
|
-
haystack.downcase.include?(value.downcase)
|
|
11
|
-
end
|
|
5
|
+
value, haystack = Textual.operands(target, config)
|
|
12
6
|
|
|
13
|
-
if
|
|
7
|
+
if Textual.include?(haystack, value, config)
|
|
14
8
|
Result.new(passed: true, detail: "contains #{value.inspect}")
|
|
15
9
|
else
|
|
16
10
|
Result.new(passed: false, detail: "does not contain #{value.inspect}")
|