completion-kit 0.28.45 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/javascripts/completion_kit/application.js +52 -23
  3. data/app/assets/stylesheets/completion_kit/application.css +56 -15
  4. data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
  5. data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
  6. data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
  7. data/app/controllers/completion_kit/metrics_controller.rb +48 -46
  8. data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
  9. data/app/controllers/completion_kit/runs_controller.rb +18 -6
  10. data/app/helpers/completion_kit/application_helper.rb +110 -52
  11. data/app/jobs/completion_kit/check_review_job.rb +27 -6
  12. data/app/models/completion_kit/metric.rb +103 -28
  13. data/app/models/completion_kit/metric_version.rb +16 -1
  14. data/app/models/completion_kit/prompt.rb +1 -1
  15. data/app/models/completion_kit/review.rb +2 -0
  16. data/app/models/completion_kit/run.rb +36 -18
  17. data/app/services/completion_kit/checks/contains.rb +2 -8
  18. data/app/services/completion_kit/checks/equals.rb +2 -7
  19. data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
  20. data/app/services/completion_kit/checks/flag.rb +13 -0
  21. data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
  22. data/app/services/completion_kit/checks/labels.rb +104 -0
  23. data/app/services/completion_kit/checks/length_bounds.rb +4 -4
  24. data/app/services/completion_kit/checks/list_overlap.rb +96 -0
  25. data/app/services/completion_kit/checks/not_contains.rb +2 -8
  26. data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
  27. data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
  28. data/app/services/completion_kit/checks/numeric_value.rb +45 -0
  29. data/app/services/completion_kit/checks/regex.rb +2 -2
  30. data/app/services/completion_kit/checks/registry.rb +107 -4
  31. data/app/services/completion_kit/checks/result.rb +5 -1
  32. data/app/services/completion_kit/checks/target_resolver.rb +43 -8
  33. data/app/services/completion_kit/checks/textual.rb +19 -0
  34. data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
  35. data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
  36. data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
  37. data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
  38. data/app/services/completion_kit/promptfoo_importer.rb +4 -4
  39. data/app/services/completion_kit/run_comparison.rb +6 -0
  40. data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
  41. data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
  42. data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
  43. data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
  44. data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
  45. data/app/views/completion_kit/metrics/_form.html.erb +62 -47
  46. data/app/views/completion_kit/onboarding/show.html.erb +2 -2
  47. data/app/views/completion_kit/prompts/_form.html.erb +2 -1
  48. data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
  49. data/app/views/completion_kit/responses/show.html.erb +1 -1
  50. data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
  51. data/app/views/completion_kit/runs/_form.html.erb +21 -11
  52. data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
  53. data/app/views/completion_kit/runs/_row.html.erb +10 -8
  54. data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
  55. data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
  56. data/app/views/completion_kit/runs/compare.html.erb +10 -6
  57. data/app/views/completion_kit/runs/edit.html.erb +1 -1
  58. data/app/views/completion_kit/runs/new.html.erb +1 -1
  59. data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
  60. data/app/views/layouts/completion_kit/application.html.erb +1 -1
  61. data/config/locales/en.yml +2 -0
  62. data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
  63. data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
  64. data/lib/completion_kit/errors.rb +2 -2
  65. data/lib/completion_kit/version.rb +1 -1
  66. metadata +10 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7c6a9a433f596def84c665f6b6fbe2b5aa4ed1422ac923f20a2a2d246aad02c4
4
- data.tar.gz: 3a0ad110587d51bad3b41f89439da3efa79c199887d58b76703b0d46d16d7f6d
3
+ metadata.gz: d1da0c4306c55de2ce239393e22ce0512a051e9c85285165fa363b746bbf4a09
4
+ data.tar.gz: e9f2adb520737b42f5b9a51841cf32b0e5249bb74b5c13bedc3758c17dbca29a
5
5
  SHA512:
6
- metadata.gz: 5f17ea2b059df5b6e2656c9a80a60d87ccd5a55b7d1d2a2599ca3b913227eca49d19a983015765b52accc0287a8e07e60dedb3e1d92d2744e81f5803a32f5b19
7
- data.tar.gz: 673b44c3f1eb7edaf963acb85a0451067ed1f6fa9ee73fee156518c031e8fb2b74786aa119c88d39281cc36e4a5e56da3ab8354a3c0411fe1eca800ed96b275e
6
+ metadata.gz: 226c925a5fa2e048acbb20d0c71bf63c02af14102073875d7c42858da7e9a36c6a6f4ad87eb570a5a18d977e25a16e104ec95e37f2a1424f1ddadd3a7a3e0c69
7
+ data.tar.gz: 64638c834849310c7f818bd2db14bf0a682447e4b52409bd5f9656ec0fe14b732a08dea84d8c414adb36e509e807b7869945271aa6ee8aca8c7396c99a78c324
@@ -8,6 +8,8 @@ document.addEventListener("turbo:load", function() {
8
8
  if (ckDiscoveringInDom()) ckStartDiscoveryPolling(0);
9
9
  });
10
10
 
11
+ document.addEventListener("turbo:render", ckAutoFocusFirstError);
12
+
11
13
  document.addEventListener("input", function(e) {
12
14
  if (!e.target || e.target.id !== "tag_name") return;
13
15
  var text = document.getElementById("tag-pill-text");
@@ -214,37 +216,56 @@ document.addEventListener("click", function(e) {
214
216
  });
215
217
  });
216
218
 
217
- var CK_CHECK_FIELDS = {
218
- contains: ["value", "case_sensitive", "trim"],
219
- not_contains: ["value", "case_sensitive", "trim"],
220
- equals: ["value", "case_sensitive", "trim"],
221
- regex: ["pattern", "case_sensitive", "multiline"],
222
- valid_json: [],
223
- json_path_equals: ["json_path", "expected"],
224
- length_bounds: ["min", "max"]
225
- };
219
+ function ckCheckEditor(scope) {
220
+ if (!scope) return null;
221
+ if (scope.matches && scope.matches("[data-ck-check-form]")) return scope;
222
+ return scope.querySelector ? scope.querySelector("[data-ck-check-form]") : null;
223
+ }
226
224
 
227
- var CK_VALUE_KINDS = ["contains", "not_contains", "equals"];
225
+ function ckCheckForm(editor) {
226
+ try {
227
+ return JSON.parse(editor.getAttribute("data-ck-check-form"));
228
+ } catch (e) {
229
+ return null;
230
+ }
231
+ }
232
+
233
+ function ckApplyCheckLabels(editor, form, kind) {
234
+ var labels = form.labels[kind] || {};
235
+ var hints = form.hints[kind] || {};
236
+ editor.querySelectorAll("[data-ck-check-label]").forEach(function(node) {
237
+ var key = node.getAttribute("data-ck-check-label");
238
+ node.textContent = labels[key] || form.defaultLabels[key] || "";
239
+ });
240
+ editor.querySelectorAll("[data-ck-check-hint]").forEach(function(node) {
241
+ var key = node.getAttribute("data-ck-check-hint");
242
+ node.textContent = hints[key] || form.defaultHints[key] || "";
243
+ });
244
+ }
228
245
 
229
246
  function ckApplyCheckFields(scope) {
230
- if (!scope) return;
231
- var kindSelect = scope.querySelector('[name="metric[check_config][check_kind]"]');
247
+ var editor = ckCheckEditor(scope);
248
+ if (!editor) return;
249
+ var form = ckCheckForm(editor);
250
+ if (!form) return;
251
+ var kindSelect = editor.querySelector('[name="metric[check_config][check_kind]"]');
232
252
  if (!kindSelect) return;
233
253
  var kind = kindSelect.value;
234
- var visible = (CK_CHECK_FIELDS[kind] || []).slice();
235
- var targetSelect = scope.querySelector('[name="metric[check_config][target]"]');
254
+ var visible = (form.fields[kind] || []).slice();
255
+ var targetSelect = editor.querySelector('[name="metric[check_config][target]"]');
236
256
  var targetIsJsonPath = !!(targetSelect && targetSelect.value === "json_path");
257
+ var expectedKey = form.expected[kind];
237
258
 
238
- if (CK_VALUE_KINDS.indexOf(kind) !== -1) {
259
+ if (expectedKey) {
239
260
  visible.push("compare_to");
240
- var comparison = scope.querySelector('[name="metric[check_config][compare_to]"]:checked');
261
+ var comparison = editor.querySelector('[name="metric[check_config][compare_to]"]:checked');
241
262
  if (comparison && comparison.value === "expected") {
242
- visible = visible.filter(function(key) { return key !== "value"; });
263
+ visible = visible.filter(function(key) { return key !== expectedKey; });
243
264
  visible.push("expected_path");
244
265
  }
245
266
  }
246
267
 
247
- scope.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
268
+ editor.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
248
269
  var key = field.getAttribute("data-ck-check-field");
249
270
  var show;
250
271
  if (key === "target_path") {
@@ -254,6 +275,8 @@ function ckApplyCheckFields(scope) {
254
275
  }
255
276
  field.hidden = !show;
256
277
  });
278
+
279
+ ckApplyCheckLabels(editor, form, kind);
257
280
  }
258
281
 
259
282
  function ckApplyMetricType(group) {
@@ -271,14 +294,17 @@ function ckApplyMetricType(group) {
271
294
  ckApplyCheckFields(scope);
272
295
  }
273
296
 
274
- document.addEventListener("turbo:load", function() {
297
+ function ckApplyMetricForms() {
275
298
  document.querySelectorAll("[data-ck-metric-type]").forEach(function(group) {
276
299
  ckApplyMetricType(group);
277
300
  });
278
- document.querySelectorAll('[data-ck-metric-editor="check"]').forEach(function(editor) {
301
+ document.querySelectorAll("[data-ck-check-form]").forEach(function(editor) {
279
302
  ckApplyCheckFields(editor);
280
303
  });
281
- });
304
+ }
305
+
306
+ document.addEventListener("turbo:load", ckApplyMetricForms);
307
+ document.addEventListener("turbo:render", ckApplyMetricForms);
282
308
 
283
309
  document.addEventListener("change", function(e) {
284
310
  var target = e.target;
@@ -305,11 +331,14 @@ function ckApplyProviderFields(scope) {
305
331
  });
306
332
  }
307
333
 
308
- document.addEventListener("turbo:load", function() {
334
+ function ckApplyProviderForms() {
309
335
  document.querySelectorAll("[data-ck-provider-select]").forEach(function(select) {
310
336
  ckApplyProviderFields(select.closest("form") || document);
311
337
  });
312
- });
338
+ }
339
+
340
+ document.addEventListener("turbo:load", ckApplyProviderForms);
341
+ document.addEventListener("turbo:render", ckApplyProviderForms);
313
342
 
314
343
  document.addEventListener("change", function(e) {
315
344
  var target = e.target;
@@ -2053,6 +2053,10 @@ label.ck-checkbox[hidden] {
2053
2053
  flex: 1;
2054
2054
  }
2055
2055
 
2056
+ .ck-field-row:not(:has(> .ck-field:not([hidden]))) {
2057
+ display: none;
2058
+ }
2059
+
2056
2060
  .ck-metric-type-chip {
2057
2061
  text-transform: uppercase;
2058
2062
  letter-spacing: 0.04em;
@@ -2060,32 +2064,42 @@ label.ck-checkbox[hidden] {
2060
2064
  }
2061
2065
 
2062
2066
  .ck-check-spec {
2063
- display: flex;
2064
- flex-direction: column;
2065
- gap: 0.4rem;
2067
+ display: grid;
2068
+ grid-template-columns: max-content minmax(0, 1fr);
2069
+ column-gap: 1rem;
2070
+ row-gap: 0.4rem;
2066
2071
  margin: 0.5rem 0 0;
2067
2072
  }
2068
2073
 
2069
2074
  .ck-check-spec__row {
2070
- display: flex;
2071
- gap: 0.75rem;
2075
+ display: grid;
2076
+ grid-column: 1 / -1;
2077
+ grid-template-columns: subgrid;
2072
2078
  align-items: baseline;
2073
2079
  }
2074
2080
 
2075
2081
  .ck-check-spec__term {
2076
- flex: 0 0 7rem;
2077
2082
  margin: 0;
2078
2083
  font-size: 0.8rem;
2079
2084
  text-transform: uppercase;
2080
2085
  letter-spacing: 0.04em;
2086
+ white-space: nowrap;
2081
2087
  color: var(--ck-muted);
2082
2088
  }
2083
2089
 
2084
2090
  .ck-check-spec__val {
2085
2091
  margin: 0;
2092
+ min-width: 0;
2093
+ overflow-wrap: anywhere;
2086
2094
  color: var(--ck-text);
2087
2095
  }
2088
2096
 
2097
+ @media (max-width: 640px) {
2098
+ .ck-check-spec { grid-template-columns: minmax(0, 1fr); row-gap: 0.6rem; }
2099
+ .ck-check-spec__row { grid-template-columns: minmax(0, 1fr); row-gap: 0.15rem; }
2100
+ .ck-check-spec__term { white-space: normal; }
2101
+ }
2102
+
2089
2103
  .ck-check-spec__code {
2090
2104
  font-family: var(--ck-mono);
2091
2105
  font-size: 0.85rem;
@@ -2098,6 +2112,7 @@ label.ck-checkbox[hidden] {
2098
2112
 
2099
2113
  .ck-select {
2100
2114
  position: relative;
2115
+ min-width: 0;
2101
2116
  }
2102
2117
 
2103
2118
  .ck-select__trigger {
@@ -2311,6 +2326,15 @@ label.ck-checkbox[hidden] {
2311
2326
  flex: 1;
2312
2327
  }
2313
2328
 
2329
+ .ck-select-with-action > .field_with_errors {
2330
+ flex: 1;
2331
+ min-width: 0;
2332
+ }
2333
+
2334
+ .ck-select-with-action > .field_with_errors > .ck-input {
2335
+ width: 100%;
2336
+ }
2337
+
2314
2338
  .ck-icon-btn:hover {
2315
2339
  background: rgba(255, 255, 255, 0.05);
2316
2340
  border-color: var(--ck-line-strong);
@@ -3262,13 +3286,13 @@ select.ck-input {
3262
3286
 
3263
3287
  .ck-run-compare-table td { vertical-align: top; }
3264
3288
 
3265
- .ck-table-scroll { overflow-x: auto; }
3289
+ .ck-table-scroll { position: relative; overflow-x: auto; }
3266
3290
 
3267
3291
  .ck-run-compare-table th { white-space: nowrap; }
3268
3292
  .ck-run-compare-table th:nth-child(1) { width: 7rem; }
3269
- .ck-run-compare-table th:nth-child(2) { width: 15rem; }
3270
- .ck-run-compare-table th:nth-child(3) { width: 12rem; }
3271
- .ck-run-compare-table th:nth-child(4) { width: 8rem; }
3293
+ .ck-run-compare-table th:nth-child(2) { width: 12rem; }
3294
+ .ck-run-compare-table th:nth-child(3) { width: 16rem; text-align: center; }
3295
+ .ck-run-compare-table th:nth-child(4) { width: 9rem; }
3272
3296
  .ck-run-compare-table th:nth-child(5) { width: auto; }
3273
3297
 
3274
3298
  .ck-run-compare-table th.ck-num,
@@ -3283,13 +3307,26 @@ select.ck-input {
3283
3307
  font-weight: 500;
3284
3308
  }
3285
3309
 
3286
- .ck-cmp-score__pair { display: inline-flex; align-items: center; gap: 0.45rem; }
3310
+ .ck-cmp-score__pair {
3311
+ display: grid;
3312
+ grid-template-columns: minmax(max-content, 1fr) auto minmax(max-content, 1fr);
3313
+ align-items: center;
3314
+ gap: 0.45rem;
3315
+ width: 100%;
3316
+ }
3317
+ .ck-cmp-score__pair > :first-child { justify-self: end; }
3318
+ .ck-cmp-score__pair > :last-child { justify-self: start; }
3319
+ @media (max-width: 640px) {
3320
+ .ck-cmp-score__pair { display: inline-grid; width: auto; }
3321
+ }
3322
+ .ck-cmp-score__pair .ck-badge { white-space: nowrap; }
3287
3323
  .ck-cmp-score__arrow { color: var(--ck-dim); font-size: 0.78rem; }
3288
3324
  .ck-cmp-score__link { display: inline-flex; text-decoration: none; border-radius: 7px; transition: box-shadow 0.12s ease; }
3289
3325
  .ck-cmp-score__link:hover { box-shadow: 0 0 0 1px var(--ck-accent); }
3290
3326
  .ck-cmp-score__none { color: var(--ck-dim); }
3291
3327
 
3292
- .ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; }
3328
+ .ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; white-space: nowrap; }
3329
+ .ck-cmp-delta__pair { display: inline-flex; align-items: baseline; justify-content: flex-end; gap: 0.35rem; }
3293
3330
  .ck-delta__arrow { font-size: 0.72rem; }
3294
3331
 
3295
3332
  .ck-cmp-ver__one, .ck-cmp-ver__pair {
@@ -5175,11 +5212,15 @@ a.tag-mark {
5175
5212
  }
5176
5213
 
5177
5214
 
5178
- .ck-input--error {
5215
+ .ck-input--error,
5216
+ .ck-input[aria-invalid="true"],
5217
+ .ck-select__trigger[aria-invalid="true"] {
5179
5218
  border-color: var(--ck-danger);
5180
- background: rgba(248, 113, 113, 0.05);
5219
+ background-color: rgba(248, 113, 113, 0.05);
5181
5220
  }
5182
- .ck-input--error:focus {
5221
+ .ck-input--error:focus,
5222
+ .ck-input[aria-invalid="true"]:focus,
5223
+ .ck-select__trigger[aria-invalid="true"]:focus-visible {
5183
5224
  box-shadow: 0 0 0 3px rgba(248, 113, 113, 0.15);
5184
5225
  border-color: var(--ck-danger);
5185
5226
  }
@@ -16,6 +16,8 @@ module CompletionKit
16
16
  def publish
17
17
  @version.publish!
18
18
  render json: @version.reload
19
+ rescue ActiveRecord::RecordInvalid => e
20
+ render_validation_errors(e.record)
19
21
  end
20
22
 
21
23
  def destroy
@@ -70,7 +70,7 @@ module CompletionKit
70
70
  def metric_params
71
71
  params.permit(:name, :instruction, :metric_type,
72
72
  rubric_bands: [:stars, :description],
73
- check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
73
+ check_config: [*%i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim], { value: [] }],
74
74
  tag_names: [])
75
75
  end
76
76
  end
@@ -112,9 +112,12 @@ module CompletionKit
112
112
  right_score: r_review ? r_review.ai_score : nil,
113
113
  left_passed: l_review&.passed,
114
114
  right_passed: r_review&.passed,
115
+ left_check_score: l_review&.check_score&.to_f,
116
+ right_check_score: r_review&.check_score&.to_f,
115
117
  left_metric_version_id: l_review&.metric_version_id,
116
118
  right_metric_version_id: r_review&.metric_version_id,
117
- delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
119
+ delta: CompletionKit::RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
120
+ check_score_delta: CompletionKit::RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
118
121
  result_change: CompletionKit::RunComparison.result_change(l_review&.passed, r_review&.passed)
119
122
  }
120
123
  end.compact
@@ -61,7 +61,9 @@ module CompletionKit
61
61
  @published_metric_version = MetricVersion.published.where(metric_id: @metric.id, current: true).first
62
62
  @improve_disagreement_count = @metric.check? ? 0 : Agreement.where(metric_id: @metric.id, verdict: "disagree", run_id: CompletionKit::Run.visible_run_ids).count
63
63
 
64
- if @edit_draft
64
+ if @edit_draft&.check?
65
+ @metric.check_config = @edit_draft.check_config
66
+ elsif @edit_draft
65
67
  @metric.instruction = @edit_draft.instruction
66
68
  @metric.rubric_bands = @edit_draft.rubric_bands
67
69
  end
@@ -78,17 +80,12 @@ module CompletionKit
78
80
  end
79
81
 
80
82
  def update
81
- meta_attrs = metric_params.except(:instruction, :rubric_bands, :check_config)
82
-
83
- unless @metric.update(meta_attrs)
84
- return render(:edit, status: :unprocessable_entity)
85
- end
83
+ attrs = metric_params
84
+ @metric.assign_attributes(attrs.except(:instruction, :rubric_bands, :check_config))
85
+ return update_check_definition(attrs.to_unsafe_h[:check_config]) if @metric.check?
86
+ return render(:edit, status: :unprocessable_entity) unless @metric.save
86
87
 
87
- if @metric.check?
88
- update_check_definition
89
- else
90
- update_judge_definition
91
- end
88
+ update_judge_definition
92
89
  end
93
90
 
94
91
  def destroy
@@ -157,7 +154,11 @@ module CompletionKit
157
154
  reverting = was_published_already && !version.current?
158
155
  previously_current = MetricVersion.current.find_by(metric_id: @metric.id)
159
156
 
160
- version.publish!
157
+ begin
158
+ version.publish!
159
+ rescue ActiveRecord::RecordInvalid => e
160
+ return redirect_to(metric_path(@metric), alert: "#{version.version_label} was not published. #{e.record.errors.full_messages.to_sentence}.")
161
+ end
161
162
 
162
163
  if reverting
163
164
  redirect_to metric_path(@metric),
@@ -207,28 +208,28 @@ module CompletionKit
207
208
  end
208
209
  end
209
210
 
210
- def update_check_definition
211
- raw = metric_params[:check_config]
212
- proposed = raw.nil? ? nil : normalize_check_config(raw)
211
+ def update_check_definition(proposed)
212
+ live = @metric.check_config
213
+ changed = !proposed.nil? && CompletionKit::Checks::Registry.canonical(proposed) != CompletionKit::Checks::Registry.canonical(live)
214
+ @metric.check_config = proposed if changed
215
+ return render(:edit, status: :unprocessable_entity) unless @metric.valid?
216
+ return save_check_draft(live, proposed) if changed && @metric.reviews.exists?
213
217
 
214
- unless !proposed.nil? && proposed != @metric.check_config
215
- return redirect_to(metric_path(@metric), notice: "Metric was successfully updated.")
216
- end
218
+ @metric.save!
219
+ MetricVersion.published.find_by(metric_id: @metric.id, current: true)&.update!(metric_type: "check", check_config: proposed) if changed
220
+ redirect_to metric_path(@metric), notice: "Metric was successfully updated."
221
+ end
217
222
 
218
- if @metric.reviews.exists?
219
- MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
220
- draft = MetricVersion.create!(
221
- metric: @metric, metric_type: "check", check_config: proposed,
222
- state: "draft", source: "edit", current: false
223
- )
224
- redirect_to edit_metric_path(@metric),
225
- notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
226
- else
227
- @metric.update!(check_config: proposed)
228
- current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
229
- current_pub&.update!(metric_type: "check", check_config: proposed)
230
- redirect_to metric_path(@metric), notice: "Metric was successfully updated."
231
- end
223
+ def save_check_draft(live, proposed)
224
+ @metric.check_config = live
225
+ @metric.save!(validate: false)
226
+ MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
227
+ draft = MetricVersion.create!(
228
+ metric: @metric, metric_type: "check", check_config: proposed,
229
+ state: "draft", source: "edit", current: false
230
+ )
231
+ redirect_to edit_metric_path(@metric),
232
+ notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
232
233
  end
233
234
 
234
235
  def set_metric
@@ -247,7 +248,7 @@ module CompletionKit
247
248
  def metric_params
248
249
  permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
249
250
  rubric_bands: [:stars, :description],
250
- check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
251
+ check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim],
251
252
  tag_names: [])
252
253
  permitted[:check_config] = normalize_check_config(permitted[:check_config]) if permitted.key?(:check_config)
253
254
  permitted
@@ -255,22 +256,23 @@ module CompletionKit
255
256
 
256
257
  def normalize_check_config(config)
257
258
  hash = config.to_unsafe_h.stringify_keys
258
- %w[min max].each { |key| hash[key] = hash[key].to_i if hash[key].present? }
259
+ %w[min max tolerance].each { |key| hash[key] = coerce_number(hash[key]) if hash[key].present? }
259
260
  %w[case_sensitive multiline trim].each { |key| hash[key] = ActiveModel::Type::Boolean.new.cast(hash[key]) if hash.key?(key) }
260
- hash["expected"] = coerce_scalar(hash["expected"]) if hash["expected"].present?
261
- if hash["compare_to"] == "expected"
262
- hash.delete("value")
263
- else
264
- hash.delete("compare_to")
265
- hash.delete("expected_path")
266
- end
267
- hash.reject { |_, value| value.nil? || value == "" }
261
+ hash["expected"] = CompletionKit::Checks::Registry.parse_json(hash["expected"]) if hash["expected"].present?
262
+ prune_check_config(hash.except(*CompletionKit::Checks::Registry.unused_keys(hash)))
268
263
  end
269
264
 
270
- def coerce_scalar(value)
271
- JSON.parse(value)
272
- rescue JSON::ParserError
273
- value
265
+ def prune_check_config(hash)
266
+ allowed = CompletionKit::Checks::Registry::SHARED_KEYS +
267
+ CompletionKit::Checks::Registry.config_keys(hash["check_kind"])
268
+ hash.slice(*allowed).reject { |_, value| value.nil? || value == "" }
269
+ end
270
+
271
+ def coerce_number(value)
272
+ number = CompletionKit::Checks::NumericValue.parse(value)
273
+ return value if number.nil?
274
+
275
+ number == number.to_i ? number.to_i : number
274
276
  end
275
277
 
276
278
  def normalize_rubric_bands_for_update(bands)
@@ -21,7 +21,7 @@ module CompletionKit
21
21
 
22
22
  def sample_data
23
23
  Onboarding::SampleData.install!
24
- redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt — edit or delete them whenever."
24
+ redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt. Edit or delete them whenever."
25
25
  end
26
26
  end
27
27
  end
@@ -40,6 +40,7 @@ module CompletionKit
40
40
  end
41
41
 
42
42
  def create
43
+ @judge_only = judge_only_chosen?
43
44
  @run = Run.new(run_attributes)
44
45
  if @run.save
45
46
  redirect_to run_path(@run), notice: "Run was successfully created."
@@ -50,10 +51,11 @@ module CompletionKit
50
51
  end
51
52
 
52
53
  def update
54
+ @judge_only = judge_only_chosen?
53
55
  if @run.responses.any? && run_generation_changed?
54
56
  attrs = run_attributes.to_h
55
57
  attrs.delete("name") if attrs["name"].to_s == @run.name.to_s
56
- new_run = Run.new(attrs.merge(status: "pending"))
58
+ new_run = Run.new(attrs.merge(status: "pending", cloned_from_judge_model: @run.judge_model))
57
59
  if new_run.save
58
60
  redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
59
61
  else
@@ -200,9 +202,12 @@ module CompletionKit
200
202
  right_score: r_review ? r_review.ai_score : nil,
201
203
  left_passed: l_review&.passed,
202
204
  right_passed: r_review&.passed,
205
+ left_check_score: l_review&.check_score&.to_f,
206
+ right_check_score: r_review&.check_score&.to_f,
203
207
  left_version_label: version_label_for(l_review, metric_versions),
204
208
  right_version_label: version_label_for(r_review, metric_versions),
205
- delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
209
+ delta: RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
210
+ check_score_delta: RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
206
211
  result_change: RunComparison.result_change(l_review&.passed, r_review&.passed)
207
212
  }
208
213
  end.compact
@@ -224,11 +229,16 @@ module CompletionKit
224
229
  end
225
230
 
226
231
  def run_params
227
- params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
232
+ params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, :judge_only, metric_ids: [], tag_names: [])
228
233
  end
229
234
 
230
235
  def run_attributes
231
- run_params.except(:metric_ids).merge(selected_metric_ids: run_params[:metric_ids])
236
+ attributes = run_params.except(:metric_ids, :judge_only).merge(selected_metric_ids: run_params[:metric_ids])
237
+ judge_only_chosen? ? attributes.merge(prompt_id: nil) : attributes
238
+ end
239
+
240
+ def judge_only_chosen?
241
+ run_params[:judge_only] == "1"
232
242
  end
233
243
 
234
244
  # Editing a run that already has results forks a new run — but only when a
@@ -237,9 +247,10 @@ module CompletionKit
237
247
  GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature max_tokens judge_temperature output_column expected_column].freeze
238
248
 
239
249
  def run_generation_changed?
250
+ attributes = run_attributes
240
251
  GENERATION_RUN_FIELDS.each do |field|
241
- next unless run_params.key?(field)
242
- return true if normalize_run_field(field, run_params[field]) != normalize_run_field(field, @run.public_send(field))
252
+ next unless attributes.key?(field)
253
+ return true if normalize_run_field(field, attributes[field]) != normalize_run_field(field, @run.public_send(field))
243
254
  end
244
255
  return false unless params[:run].key?(:metric_ids)
245
256
  Array(params[:run][:metric_ids]).map(&:to_i).reject(&:zero?).sort != @run.metric_ids.sort
@@ -247,6 +258,7 @@ module CompletionKit
247
258
 
248
259
  def normalize_run_field(field, value)
249
260
  s = value.to_s.strip
261
+ return s.presence || "actual_output" if field == :output_column
250
262
  return nil if s.empty?
251
263
  case field
252
264
  when :temperature, :judge_temperature then s.to_f