completion-kit 0.28.45 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/javascripts/completion_kit/application.js +52 -23
  3. data/app/assets/stylesheets/completion_kit/application.css +56 -15
  4. data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
  5. data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
  6. data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
  7. data/app/controllers/completion_kit/metrics_controller.rb +48 -46
  8. data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
  9. data/app/controllers/completion_kit/runs_controller.rb +18 -6
  10. data/app/helpers/completion_kit/application_helper.rb +110 -52
  11. data/app/jobs/completion_kit/check_review_job.rb +27 -6
  12. data/app/models/completion_kit/metric.rb +103 -28
  13. data/app/models/completion_kit/metric_version.rb +16 -1
  14. data/app/models/completion_kit/prompt.rb +1 -1
  15. data/app/models/completion_kit/review.rb +2 -0
  16. data/app/models/completion_kit/run.rb +36 -18
  17. data/app/services/completion_kit/checks/contains.rb +2 -8
  18. data/app/services/completion_kit/checks/equals.rb +2 -7
  19. data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
  20. data/app/services/completion_kit/checks/flag.rb +13 -0
  21. data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
  22. data/app/services/completion_kit/checks/labels.rb +104 -0
  23. data/app/services/completion_kit/checks/length_bounds.rb +4 -4
  24. data/app/services/completion_kit/checks/list_overlap.rb +96 -0
  25. data/app/services/completion_kit/checks/not_contains.rb +2 -8
  26. data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
  27. data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
  28. data/app/services/completion_kit/checks/numeric_value.rb +45 -0
  29. data/app/services/completion_kit/checks/regex.rb +2 -2
  30. data/app/services/completion_kit/checks/registry.rb +107 -4
  31. data/app/services/completion_kit/checks/result.rb +5 -1
  32. data/app/services/completion_kit/checks/target_resolver.rb +43 -8
  33. data/app/services/completion_kit/checks/textual.rb +19 -0
  34. data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
  35. data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
  36. data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
  37. data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
  38. data/app/services/completion_kit/promptfoo_importer.rb +4 -4
  39. data/app/services/completion_kit/run_comparison.rb +6 -0
  40. data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
  41. data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
  42. data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
  43. data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
  44. data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
  45. data/app/views/completion_kit/metrics/_form.html.erb +62 -47
  46. data/app/views/completion_kit/onboarding/show.html.erb +2 -2
  47. data/app/views/completion_kit/prompts/_form.html.erb +2 -1
  48. data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
  49. data/app/views/completion_kit/responses/show.html.erb +1 -1
  50. data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
  51. data/app/views/completion_kit/runs/_form.html.erb +21 -11
  52. data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
  53. data/app/views/completion_kit/runs/_row.html.erb +10 -8
  54. data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
  55. data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
  56. data/app/views/completion_kit/runs/compare.html.erb +10 -6
  57. data/app/views/completion_kit/runs/edit.html.erb +1 -1
  58. data/app/views/completion_kit/runs/new.html.erb +1 -1
  59. data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
  60. data/app/views/layouts/completion_kit/application.html.erb +1 -1
  61. data/config/locales/en.yml +2 -0
  62. data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
  63. data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
  64. data/lib/completion_kit/errors.rb +2 -2
  65. data/lib/completion_kit/version.rb +1 -1
  66. metadata +10 -1
@@ -151,18 +151,22 @@ module CompletionKit
151
151
  model[:name]
152
152
  end
153
153
 
154
- def ck_grouped_models(models, selected = nil)
154
+ def ck_grouped_models(models, selected = nil, rejected: false)
155
+ kept = nil
155
156
  if selected.present? && models.none? { |m| m[:id] == selected }
156
- retired = CompletionKit::Model.find_by(model_id: selected)
157
+ retired = CompletionKit::Model.find_by(model_id: selected) unless rejected
157
158
  if retired
158
159
  models = models + [{ id: retired.model_id, name: "#{retired.display_name || retired.model_id} (retired)", provider: retired.provider }]
160
+ else
161
+ kept = [["#{selected} (unavailable)", selected]]
159
162
  end
160
163
  end
161
164
 
162
165
  groups = models.group_by { |m| ck_model_optgroup_label(m) }
163
166
  ordered_keys = groups.keys.sort_by { |label| ck_model_optgroup_sort_key(label) }
164
167
  grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [ck_model_option_label(m), m[:id]] }] }
165
- grouped_options_for_select(grouped, selected)
168
+ options = grouped_options_for_select(grouped, selected)
169
+ kept ? options_for_select(kept, selected) + options : options
166
170
  end
167
171
 
168
172
  # Optgroup label for the model select — mirrors the provider models table:
@@ -170,15 +174,15 @@ module CompletionKit
170
174
  # else is a single group.
171
175
  def ck_model_optgroup_label(model)
172
176
  case model[:provider]
173
- when "openrouter" then "OpenRouter — #{model[:id].to_s.split("/", 2).first.delete_prefix("~")}"
174
- when "openai" then "OpenAI — #{ck_openai_model_family(model[:id])}"
177
+ when "openrouter" then "OpenRouter · #{model[:id].to_s.split("/", 2).first.delete_prefix("~")}"
178
+ when "openai" then "OpenAI · #{ck_openai_model_family(model[:id])}"
175
179
  else ck_provider_label(model[:provider])
176
180
  end
177
181
  end
178
182
 
179
183
  def ck_model_optgroup_sort_key(label)
180
- if label.start_with?("OpenAI — ")
181
- [0, OPENAI_MODEL_FAMILY_ORDER.index(label.delete_prefix("OpenAI — ")), label]
184
+ if label.start_with?("OpenAI · ")
185
+ [0, OPENAI_MODEL_FAMILY_ORDER.index(label.delete_prefix("OpenAI · ")), label]
182
186
  elsif label.start_with?("OpenRouter")
183
187
  [2, 0, label]
184
188
  else
@@ -200,11 +204,52 @@ module CompletionKit
200
204
  :low
201
205
  end
202
206
 
203
- def ck_check_badge(passed)
207
+ def ck_percent_points(fraction)
208
+ (BigDecimal(fraction.to_s) * 100).round
209
+ end
210
+
211
+ def ck_percent(fraction)
212
+ "#{ck_percent_points(fraction)}%"
213
+ end
214
+
215
+ def ck_check_verdict(passed, score = nil)
216
+ verdict = passed ? "Pass" : "Fail"
217
+ score.nil? ? verdict : "#{verdict} #{ck_percent(score)}"
218
+ end
219
+
220
+ def ck_check_average(summary)
221
+ ck_percent(summary[:avg_check_score] || summary[:pass_rate])
222
+ end
223
+
224
+ def ck_check_average_label(name, summary)
225
+ pass_rate = "#{ck_percent(summary[:pass_rate])} pass rate"
226
+ return "#{name} #{pass_rate}" unless summary[:avg_check_score]
227
+
228
+ "#{name} average score #{ck_percent(summary[:avg_check_score])}, #{pass_rate}"
229
+ end
230
+
231
+ def ck_check_score_change(left, right)
232
+ return nil if left.nil? || right.nil?
233
+
234
+ ck_percent_points(right) - ck_percent_points(left)
235
+ end
236
+
237
+ def ck_delta_chip(delta, unit = "")
238
+ tone, arrow = if delta.positive? then %w[positive ▲]
239
+ elsif delta.negative? then %w[negative ▼]
240
+ else %w[zero =]
241
+ end
242
+ content_tag(:span, class: "ck-delta ck-delta--#{tone}") do
243
+ content_tag(:span, arrow, class: "ck-delta__arrow", "aria-hidden": "true") + "#{"+" if delta.positive?}#{delta}#{unit}"
244
+ end
245
+ end
246
+
247
+ def ck_check_badge(passed, score = nil)
248
+ suffix = score.nil? ? "" : " #{ck_percent(score)}"
204
249
  if passed == true
205
- content_tag(:span, "Pass", class: ck_badge_classes(:high))
250
+ content_tag(:span, "Pass#{suffix}", class: ck_badge_classes(:high))
206
251
  elsif passed == false
207
- content_tag(:span, "Fail", class: ck_badge_classes(:low))
252
+ content_tag(:span, "Fail#{suffix}", class: ck_badge_classes(:low))
208
253
  else
209
254
  content_tag(:span, "Pending", class: ck_badge_classes(:pending))
210
255
  end
@@ -228,58 +273,71 @@ module CompletionKit
228
273
  {heading: heading, judge_model: judge ? judge_model.presence : nil}
229
274
  end
230
275
 
231
- CHECK_KIND_LABELS = {
232
- "contains" => "Contains a phrase",
233
- "not_contains" => "Does not contain a phrase",
234
- "equals" => "Equals exactly",
235
- "regex" => "Matches a pattern",
236
- "valid_json" => "Is valid JSON",
237
- "json_path_equals" => "A JSON field equals a value",
238
- "length_bounds" => "Length is within a range"
239
- }.freeze
240
-
241
- CHECK_COMPARE_TO_LABELS = {
242
- "constant" => "A value you type",
243
- "expected" => "Each row's expected value"
244
- }.freeze
245
-
246
- CHECK_TARGET_LABELS = {
247
- "response_text" => "The response text",
248
- "input_data" => "The input row",
249
- "json_path" => "A value from the response JSON"
250
- }.freeze
251
-
252
- CHECK_FIELD_LABELS = {
253
- "value" => "Text to look for",
254
- "pattern" => "Pattern",
255
- "json_path" => "JSON path",
256
- "expected" => "Expected value",
257
- "expected_path" => "Field in the answer key",
258
- "compare_to" => "Compare against",
259
- "target_path" => "Path into the JSON",
260
- "min" => "Shortest allowed",
261
- "max" => "Longest allowed",
262
- "case_sensitive" => "Case sensitive",
263
- "multiline" => "Multiline",
264
- "trim" => "Trim whitespace"
265
- }.freeze
266
-
267
276
  def ck_check_kind_label(kind)
268
- CHECK_KIND_LABELS.fetch(kind.to_s) { kind.to_s.humanize }
277
+ CompletionKit::Checks::Labels.kind(kind)
269
278
  end
270
279
 
271
280
  def ck_check_target_label(target)
272
- CHECK_TARGET_LABELS.fetch(target.to_s) { target.to_s.humanize }
281
+ CompletionKit::Checks::Labels.target(target)
273
282
  end
274
283
 
275
- def ck_check_field_label(key)
276
- CHECK_FIELD_LABELS.fetch(key.to_s) { key.to_s.humanize }
284
+ def ck_check_field_label(key, kind = nil)
285
+ CompletionKit::Checks::Labels.field(key, kind)
286
+ end
287
+
288
+ def ck_check_field_hint(key, kind = nil)
289
+ CompletionKit::Checks::Labels.hint(key, kind)
277
290
  end
278
291
 
279
292
  def ck_check_field_value(key, value)
280
- return CHECK_COMPARE_TO_LABELS.fetch(value.to_s, value) if key.to_s == "compare_to"
293
+ CompletionKit::Checks::Labels.value(key, value)
294
+ end
295
+
296
+ def ck_check_input_value(value)
297
+ value.is_a?(Array) || value.is_a?(Hash) ? value.to_json : value
298
+ end
299
+
300
+ def ck_check_expected_input_value(value)
301
+ return value if value.nil? || (value.is_a?(String) && CompletionKit::Checks::Registry.parse_json(value) == value)
302
+
303
+ value.to_json
304
+ end
305
+
306
+ def ck_check_spec_fields(config)
307
+ shown = config.except("check_kind", "target").reject do |key, _value|
308
+ CompletionKit::Checks::Registry::FLAG_KEYS.include?(key) && !CompletionKit::Checks::Flag.on?(config, key)
309
+ end
310
+ CompletionKit::Checks::Labels.in_form_order(shown)
311
+ end
312
+
313
+ def ck_check_spec_value(key, value)
314
+ return "Yes" if CompletionKit::Checks::Registry::FLAG_KEYS.include?(key.to_s)
315
+ return ck_check_field_value(key, value) if CompletionKit::Checks::Labels.worded?(key)
316
+
317
+ shown = key.to_s == "expected" ? ck_check_expected_input_value(value) : ck_check_input_value(value)
318
+ content_tag(:code, shown.to_s, class: "ck-check-spec__code")
319
+ end
320
+
321
+ def ck_check_field_invalid?(metric, key)
322
+ metric.check_error_keys.include?(key)
323
+ end
324
+
325
+ def ck_check_field_aria(metric, key)
326
+ ck_check_field_invalid?(metric, key) ? { "aria-invalid" => "true" } : {}
327
+ end
328
+
329
+ def ck_check_score_by_options
330
+ CompletionKit::Checks::Labels::SCORE_BY.to_a
331
+ end
281
332
 
282
- value
333
+ def ck_check_form_data
334
+ labels = CompletionKit::Checks::Labels
335
+ {
336
+ fields: CompletionKit::Checks::Registry::CONFIG_KEYS,
337
+ expected: CompletionKit::Checks::Registry::EXPECTED_KEYS,
338
+ labels: labels::FIELDS_BY_KIND, defaultLabels: labels::FIELDS,
339
+ hints: labels::HINTS_BY_KIND, defaultHints: labels::HINTS
340
+ }
283
341
  end
284
342
 
285
343
  def ck_result_change_badge(change)
@@ -25,6 +25,7 @@ module CompletionKit
25
25
  status: "succeeded",
26
26
  passed: result.passed,
27
27
  ai_score: nil,
28
+ check_score: result.score,
28
29
  ai_feedback: result.detail,
29
30
  error_provider: nil, error_class: nil, error_status: nil, error_message: nil
30
31
  )
@@ -36,23 +37,43 @@ module CompletionKit
36
37
  private
37
38
 
38
39
  def evaluate(response, config)
39
- target_value = Checks::TargetResolver.call(response, config)
40
+ kind = config["check_kind"]
41
+ target_value = resolve_target(response, config, kind)
40
42
  if target_value.equal?(Checks::TargetResolver::UNRESOLVED)
41
- return Checks::Result.new(passed: false, detail: "could not resolve target")
43
+ return ungraded(kind, "could not resolve target")
42
44
  end
43
45
 
44
- kind = config["check_kind"]
45
46
  if config["compare_to"] == "expected" && Checks::Registry.compares_value?(kind)
46
- expected_value = Checks::ExpectedResolver.call(response, config)
47
+ expected_value = resolve_expected(response, config, kind)
47
48
  if expected_value.equal?(Checks::ExpectedResolver::UNRESOLVED)
48
- return Checks::Result.new(passed: false, detail: "no expected value for this row")
49
+ return ungraded(kind, "no expected value for this row")
49
50
  end
50
- config = config.merge("value" => expected_value)
51
+ config = config.merge(Checks::Registry.expected_key(kind) => expected_value)
51
52
  end
52
53
 
53
54
  Checks::Registry.fetch(kind).call(target_value, config)
54
55
  end
55
56
 
57
+ def ungraded(kind, detail)
58
+ Checks::Result.new(passed: false, detail: detail, score: Checks::Registry.scores?(kind) ? 0.0 : nil)
59
+ end
60
+
61
+ def resolve_target(response, config, kind)
62
+ if Checks::Registry.raw_target?(kind)
63
+ Checks::TargetResolver.call_value(response, config)
64
+ else
65
+ Checks::TargetResolver.call(response, config)
66
+ end
67
+ end
68
+
69
+ def resolve_expected(response, config, kind)
70
+ if Checks::Registry.raw_expected?(kind)
71
+ Checks::ExpectedResolver.call_value(response, config)
72
+ else
73
+ Checks::ExpectedResolver.call(response, config)
74
+ end
75
+ end
76
+
56
77
  def record_terminal_failure!(error)
57
78
  response = Response.find_by(id: @response_id)
58
79
  return unless response
@@ -30,6 +30,7 @@ module CompletionKit
30
30
  validate :metric_type_immutable_once_in_use, on: :update
31
31
 
32
32
  before_validation :generate_key
33
+ before_validation :default_check_target, if: :check?
33
34
  before_validation :normalize_rubric_bands, if: :llm_judge?
34
35
  before_validation :set_defaults, if: :llm_judge?
35
36
 
@@ -94,6 +95,10 @@ module CompletionKit
94
95
  RunMetric.exists?(metric_id: id) || reviews.exists? || metric_versions.exists?
95
96
  end
96
97
 
98
+ def check_error_keys
99
+ errors.objects.select { |error| error.attribute == :check_config }.flat_map { |error| Array(error.options[:check_keys]) }.uniq
100
+ end
101
+
97
102
  def as_json(options = {})
98
103
  base = {
99
104
  id: id, name: name, key: key, metric_type: metric_type,
@@ -129,67 +134,137 @@ module CompletionKit
129
134
 
130
135
  kind = config["check_kind"]
131
136
  unless CompletionKit::Checks::Registry.kinds.include?(kind)
132
- errors.add(:check_config, "check_kind must be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}")
137
+ add_check_error("needs #{check_label("check_kind")} to be one of #{CompletionKit::Checks::Registry.kinds.join(", ")}", "check_kind")
133
138
  return
134
139
  end
135
140
 
136
141
  validate_check_target(config)
137
142
  validate_check_compare_to(config, kind)
138
143
  validate_check_required_keys(config, kind)
139
- validate_check_kind_rules(config, kind)
144
+ if refused_list_value?(config, kind)
145
+ add_check_error("needs #{check_label("value", kind)} to be a single value, since only list_overlap takes a list", "value")
146
+ else
147
+ validate_check_kind_rules(config, kind)
148
+ end
149
+ end
150
+
151
+ def refused_list_value?(config, kind)
152
+ config["value"].is_a?(Array) && kind != "list_overlap" && will_save_change_to_check_config?
140
153
  end
141
154
 
142
155
  def validate_check_compare_to(config, kind)
143
156
  return unless config.key?("compare_to")
144
157
 
145
158
  unless %w[constant expected].include?(config["compare_to"])
146
- errors.add(:check_config, "compare_to must be constant or expected")
159
+ add_check_error("needs #{check_label("compare_to")} to be constant or expected", "compare_to")
147
160
  return
148
161
  end
149
162
  if config["compare_to"] == "expected" && !CompletionKit::Checks::Registry.compares_value?(kind)
150
- errors.add(:check_config, "compare_to expected only applies to contains, not_contains, or equals")
163
+ add_check_error("can grade against \"Each row's expected value\" only with #{CompletionKit::Checks::Registry.comparable_kinds.join(", ")}", "compare_to")
151
164
  end
152
165
  end
153
166
 
154
167
  def validate_check_target(config)
155
168
  target = config["target"].presence || "response_text"
156
- unless CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
157
- errors.add(:check_config, "target must be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}")
158
- end
159
- if target == "json_path" && config["target_path"].to_s.strip.empty?
160
- errors.add(:check_config, "target_path is required when target is json_path")
169
+ path_given = config["target_path"].present?
170
+ if !CompletionKit::Checks::TargetResolver::TARGETS.include?(target)
171
+ add_check_error("needs #{check_label("target")} to be one of #{CompletionKit::Checks::TargetResolver::TARGETS.join(", ")}", "target")
172
+ elsif target == "json_path" && !path_given
173
+ add_check_error("needs #{check_label("target_path")} filled in to check a value from the response JSON", "target_path")
174
+ elsif target != "json_path" && path_given && will_save_change_to_check_config?
175
+ add_check_error("needs #{check_label("target")} to be \"#{CompletionKit::Checks::Labels.target("json_path")}\" when #{check_label("target_path")} is filled in", "target")
161
176
  end
162
177
  end
163
178
 
179
+ def default_check_target
180
+ config = check_config
181
+ return unless config.is_a?(Hash) && config["target"].blank? && config["target_path"].present? && will_save_change_to_check_config?
182
+
183
+ self.check_config = config.merge("target" => "json_path")
184
+ end
185
+
164
186
  def validate_check_required_keys(config, kind)
165
187
  required = CompletionKit::Checks::Registry.required_keys.fetch(kind)
166
- required -= %w[value] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
188
+ required -= [CompletionKit::Checks::Registry.expected_key(kind)] if config["compare_to"] == "expected" && CompletionKit::Checks::Registry.compares_value?(kind)
167
189
  required.each do |required_key|
168
- if required_key == "expected"
169
- errors.add(:check_config, "expected is required") unless config.key?("expected")
170
- elsif config[required_key].to_s.strip.empty?
171
- errors.add(:check_config, "#{required_key} is required")
172
- end
190
+ missing = required_key == "expected" ? !config.key?("expected") : config[required_key].to_s.strip.empty?
191
+ add_check_error("needs #{check_label(required_key, kind)} filled in", required_key) if missing
173
192
  end
174
193
  end
175
194
 
176
195
  def validate_check_kind_rules(config, kind)
177
196
  case kind
178
197
  when "regex"
179
- begin
180
- Regexp.new(config["pattern"].to_s)
181
- rescue RegexpError
182
- errors.add(:check_config, "pattern is not a valid regular expression")
183
- end
184
- when "length_bounds"
185
- min = config["min"]
186
- max = config["max"]
187
- if min.nil? && max.nil?
188
- errors.add(:check_config, "length_bounds requires at least one of min or max")
189
- elsif min && max && min.to_i > max.to_i
190
- errors.add(:check_config, "min must be less than or equal to max")
191
- end
198
+ validate_check_pattern(config)
199
+ when "list_overlap"
200
+ validate_check_list_overlap(config)
201
+ when "numeric_equals"
202
+ validate_check_numeric_equals(config)
192
203
  end
204
+ validate_check_bounds(config, kind) if CompletionKit::Checks::Registry.bounded?(kind)
205
+ end
206
+
207
+ def validate_check_pattern(config)
208
+ Regexp.new(config["pattern"].to_s)
209
+ rescue RegexpError
210
+ add_check_error("needs #{check_label("pattern")} to be a valid regular expression", "pattern")
211
+ end
212
+
213
+ def validate_check_bounds(config, kind)
214
+ min = config["min"].to_s.strip
215
+ max = config["max"].to_s.strip
216
+ min_label = check_label("min", kind)
217
+ max_label = check_label("max", kind)
218
+ if min.empty? && max.empty?
219
+ add_check_error("needs #{min_label} or #{max_label} filled in", "min", "max")
220
+ return
221
+ end
222
+
223
+ min_value = CompletionKit::Checks::NumericValue.parse(min)
224
+ max_value = CompletionKit::Checks::NumericValue.parse(max)
225
+ not_numbers = { "min" => !min.empty? && min_value.nil?, "max" => !max.empty? && max_value.nil? }.select { |_key, bad| bad }.keys
226
+ if not_numbers.any?
227
+ add_check_error("needs #{min_label} and #{max_label} to be numbers", *not_numbers)
228
+ elsif min_value && max_value && min_value > max_value
229
+ add_check_error("needs #{min_label} to be no more than #{max_label}", "min", "max")
230
+ end
231
+ end
232
+
233
+ def validate_check_list_overlap(config)
234
+ measures = CompletionKit::Checks::ListOverlap::MEASURES
235
+ if config["score_by"].present? && !measures.include?(config["score_by"])
236
+ add_check_error("needs #{check_label("score_by")} to be one of #{measures.join(", ")}", "score_by")
237
+ end
238
+ return if config["min"].to_s.strip.empty?
239
+
240
+ threshold = CompletionKit::Checks::NumericValue.parse(config["min"])
241
+ if threshold.nil? || threshold.negative? || threshold > 1
242
+ add_check_error("needs #{check_label("min", "list_overlap")} to be a number between 0 and 1", "min")
243
+ end
244
+ end
245
+
246
+ def validate_check_numeric_equals(config)
247
+ validate_check_tolerance(config)
248
+ return if config["compare_to"] == "expected" || config["value"].to_s.strip.empty?
249
+
250
+ add_check_error("needs #{check_label("value", "numeric_equals")} to be a number", "value") if CompletionKit::Checks::NumericValue.parse(config["value"]).nil?
251
+ end
252
+
253
+ def validate_check_tolerance(config)
254
+ return if config["tolerance"].to_s.strip.empty?
255
+
256
+ tolerance = CompletionKit::Checks::NumericEquals.tolerance_value(config["tolerance"])
257
+ if tolerance.nil? || tolerance.negative?
258
+ add_check_error("needs #{check_label("tolerance")} to be a number that is not negative, or a percentage like 2%", "tolerance")
259
+ end
260
+ end
261
+
262
+ def add_check_error(message, *keys)
263
+ errors.add(:check_config, message, check_keys: keys)
264
+ end
265
+
266
+ def check_label(key, kind = nil)
267
+ "\"#{CompletionKit::Checks::Labels.field(key, kind)}\""
193
268
  end
194
269
 
195
270
  def set_defaults
@@ -85,12 +85,14 @@ module CompletionKit
85
85
  end
86
86
 
87
87
  def publish!
88
+ config = publishable_check_config if check?
89
+ validate_check_config_on_metric!(config) if check?
88
90
  MetricVersion.transaction do
89
91
  self.class.where(metric_id: metric_id).where.not(id: id).update_all(current: false)
90
92
  reload
91
93
  update!(state: "published", current: true, published_at: published_at || Time.current)
92
94
  if check?
93
- metric.update_columns(metric_type: "check", check_config: check_config)
95
+ metric.update_columns(metric_type: "check", check_config: config)
94
96
  else
95
97
  metric.update_columns(
96
98
  metric_type: "llm_judge",
@@ -123,6 +125,19 @@ module CompletionKit
123
125
 
124
126
  private
125
127
 
128
+ def publishable_check_config
129
+ config = check_config.to_h.stringify_keys
130
+ return config if config["target_path"].blank?
131
+ return config.merge("target" => "json_path") if config["target"].blank?
132
+
133
+ config["target"] == "json_path" ? config : config.except("target_path")
134
+ end
135
+
136
+ def validate_check_config_on_metric!(config)
137
+ metric.assign_attributes(metric_type: "check", check_config: config)
138
+ raise ActiveRecord::RecordInvalid, metric unless metric.valid?
139
+ end
140
+
126
141
  def check_change_summary_against(previous)
127
142
  return nil if check_config == previous.check_config
128
143
 
@@ -44,7 +44,7 @@ module CompletionKit
44
44
  end
45
45
 
46
46
  def display_name
47
- "#{name} — #{version_label}"
47
+ "#{name} · #{version_label}"
48
48
  end
49
49
 
50
50
  def llm_model_provider
@@ -10,6 +10,7 @@ module CompletionKit
10
10
  validates :metric_name, presence: true
11
11
  validates :metric_version, presence: true
12
12
  validates :ai_score, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 5 }, allow_nil: true
13
+ validates :check_score, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 1 }, allow_nil: true
13
14
 
14
15
  before_validation :set_default_status
15
16
 
@@ -36,6 +37,7 @@ module CompletionKit
36
37
  id: id, response_id: response_id, metric_id: metric_id,
37
38
  metric_version_id: metric_version_id,
38
39
  metric_name: metric_name, ai_score: ai_score, passed: passed,
40
+ check_score: check_score&.to_f,
39
41
  ai_feedback: ai_feedback, status: status, attempts: attempts,
40
42
  error: error_payload
41
43
  }
@@ -10,7 +10,7 @@ module CompletionKit
10
10
  status: "pending",
11
11
  attempts: 0,
12
12
  error_provider: nil, error_class: nil, error_status: nil, error_message: nil,
13
- ai_score: nil, passed: nil, ai_feedback: nil
13
+ ai_score: nil, passed: nil, check_score: nil, ai_feedback: nil
14
14
  }.freeze
15
15
 
16
16
  belongs_to :prompt, optional: true
@@ -79,7 +79,9 @@ module CompletionKit
79
79
  Arel.sql("COUNT(ai_score)"),
80
80
  Arel.sql("SUM(CASE WHEN ai_score < #{low_score_ceiling} THEN 1 ELSE 0 END)"),
81
81
  Arel.sql("COUNT(passed)"),
82
- Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)")
82
+ Arel.sql("SUM(CASE WHEN passed THEN 1 ELSE 0 END)"),
83
+ Arel.sql("AVG(check_score)"),
84
+ Arel.sql("COUNT(check_score)")
83
85
  )
84
86
  metrics_by_run = metric_rows.group_by(&:first)
85
87
 
@@ -90,12 +92,12 @@ module CompletionKit
90
92
  run.avg_score = stats && stats[:avg] ? stats[:avg].to_f.round(2) : nil
91
93
  run.check_pass_rate = stats && stats[:resolved] > 0 ? (stats[:passed].to_f / stats[:resolved]).round(2) : nil
92
94
 
93
- run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed)|
95
+ run.metric_averages = (metrics_by_run[run.id] || []).filter_map do |(_rid, name, avg, scored, low, resolved, passed, fraction_avg, fraction_count)|
94
96
  if scored.to_i > 0
95
97
  {name: name, avg: avg.to_f.round(1), count: scored.to_i, low_count: low.to_i}
96
98
  elsif resolved.to_i > 0
97
- {name: name, kind: "check", pass_rate: (passed.to_i.to_f / resolved.to_i).round(2),
98
- count: resolved.to_i, low_count: resolved.to_i - passed.to_i}
99
+ check_summary(name, passed.to_i, resolved.to_i,
100
+ fraction_count.to_i > 0 ? fraction_avg.to_f : nil)
99
101
  end
100
102
  end
101
103
  end
@@ -109,6 +111,14 @@ module CompletionKit
109
111
  CompletionKit.config.medium_quality_threshold.to_f
110
112
  end
111
113
 
114
+ def self.check_summary(name, passed, resolved, fraction_avg)
115
+ summary = { name: name, kind: "check", pass_rate: (passed.to_f / resolved).round(2),
116
+ count: resolved, low_count: resolved - passed }
117
+ return summary if fraction_avg.nil?
118
+
119
+ summary.merge(avg_check_score: fraction_avg.round(3))
120
+ end
121
+
112
122
  TOP_SCORE = 5.0
113
123
  CEILING_MEAN = 4.8
114
124
  CEILING_TOP_SHARE = 0.9
@@ -271,9 +281,9 @@ module CompletionKit
271
281
  resolved = reviews.reject { |r| r.passed.nil? }
272
282
  next if resolved.empty?
273
283
 
274
- passed = resolved.count { |r| r.passed == true }
275
- { name: name, kind: "check", pass_rate: (passed.to_f / resolved.length).round(2),
276
- count: resolved.length, low_count: resolved.length - passed }
284
+ fractions = reviews.filter_map { |r| r.check_score&.to_f }
285
+ self.class.check_summary(name, resolved.count { |r| r.passed == true }, resolved.length,
286
+ fractions.any? ? fractions.sum / fractions.length : nil)
277
287
  end
278
288
  end
279
289
  end
@@ -333,13 +343,10 @@ module CompletionKit
333
343
  unless judge_only_input_data_checks? || (dataset && dataset.headers.include?(column))
334
344
  return fail_with_summary!("Dataset has no \"#{column}\" column")
335
345
  end
336
- else
337
- generation_error = generation_client_error
338
- return fail_with_summary!(generation_error) if generation_error
339
346
  end
340
347
 
341
- judge_error = judge_model_error
342
- return fail_with_summary!(judge_error) if judge_error
348
+ model_error = model_start_error
349
+ return fail_with_summary!(model_error) if model_error
343
350
 
344
351
  begin
345
352
  transaction do
@@ -449,6 +456,7 @@ module CompletionKit
449
456
  metric_version_id: nil,
450
457
  ai_score: nil,
451
458
  passed: nil,
459
+ check_score: nil,
452
460
  ai_feedback: nil,
453
461
  error_provider: nil,
454
462
  error_class: nil,
@@ -490,7 +498,8 @@ module CompletionKit
490
498
  end
491
499
 
492
500
  def rerun!
493
- ensure_models_resolve!
501
+ ensure_models_ready!
502
+
494
503
  Run.create!(
495
504
  prompt_id: prompt_id,
496
505
  dataset_id: dataset_id,
@@ -712,11 +721,20 @@ module CompletionKit
712
721
  %(No configured provider serves judge model "#{judge_model}")
713
722
  end
714
723
 
715
- def ensure_models_resolve!
716
- raise UnknownModelError, prompt.llm_model unless judge_only? || ApiConfig.provider_for_model(prompt.llm_model)
724
+ def model_start_error
725
+ (generation_client_error unless judge_only?) || judge_model_error
726
+ end
727
+
728
+ def ensure_models_ready!
729
+ unless judge_only?
730
+ raise UnknownModelError, prompt.llm_model unless ApiConfig.provider_for_model(prompt.llm_model)
731
+
732
+ client_error = generation_client_error
733
+ raise ConfigurationError, client_error if client_error
734
+ end
717
735
 
718
736
  judge_error = judge_model_error
719
- raise ConfigurationError, judge_error if judge_error
737
+ raise UnknownModelError.new(judge_model, judge_error) if judge_error
720
738
  end
721
739
 
722
740
  def judge_model_newly_selected?
@@ -775,7 +793,7 @@ module CompletionKit
775
793
 
776
794
  if prompt.present?
777
795
  count = Run.where(prompt_id: prompt_id).count + 1
778
- self.name = "#{prompt.name} — v#{prompt.version_number} ##{count}"
796
+ self.name = "#{prompt.name} · v#{prompt.version_number} ##{count}"
779
797
  elsif dataset.present?
780
798
  count = Run.where(prompt_id: nil, dataset_id: dataset.id).count + 1
781
799
  self.name = "#{dataset.name} scoring ##{count}"
@@ -2,15 +2,9 @@ module CompletionKit
2
2
  module Checks
3
3
  class Contains
4
4
  def call(target, config)
5
- value = config["value"].to_s
6
- haystack = target.to_s
7
- present = if config["case_sensitive"] == true
8
- haystack.include?(value)
9
- else
10
- haystack.downcase.include?(value.downcase)
11
- end
5
+ value, haystack = Textual.operands(target, config)
12
6
 
13
- if present
7
+ if Textual.include?(haystack, value, config)
14
8
  Result.new(passed: true, detail: "contains #{value.inspect}")
15
9
  else
16
10
  Result.new(passed: false, detail: "does not contain #{value.inspect}")