completion-kit 0.28.45 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/completion_kit/application.js +52 -23
- data/app/assets/stylesheets/completion_kit/application.css +56 -15
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
- data/app/controllers/completion_kit/runs_controller.rb +18 -6
- data/app/helpers/completion_kit/application_helper.rb +110 -52
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +1 -1
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +36 -18
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +2 -2
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
- data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
- data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
- data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/onboarding/show.html.erb +2 -2
- data/app/views/completion_kit/prompts/_form.html.erb +2 -1
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +21 -11
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
- data/app/views/completion_kit/runs/_row.html.erb +10 -8
- data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/edit.html.erb +1 -1
- data/app/views/completion_kit/runs/new.html.erb +1 -1
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +1 -1
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
- data/lib/completion_kit/errors.rb +2 -2
- data/lib/completion_kit/version.rb +1 -1
- metadata +10 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d1da0c4306c55de2ce239393e22ce0512a051e9c85285165fa363b746bbf4a09
|
|
4
|
+
data.tar.gz: e9f2adb520737b42f5b9a51841cf32b0e5249bb74b5c13bedc3758c17dbca29a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 226c925a5fa2e048acbb20d0c71bf63c02af14102073875d7c42858da7e9a36c6a6f4ad87eb570a5a18d977e25a16e104ec95e37f2a1424f1ddadd3a7a3e0c69
|
|
7
|
+
data.tar.gz: 64638c834849310c7f818bd2db14bf0a682447e4b52409bd5f9656ec0fe14b732a08dea84d8c414adb36e509e807b7869945271aa6ee8aca8c7396c99a78c324
|
|
@@ -8,6 +8,8 @@ document.addEventListener("turbo:load", function() {
|
|
|
8
8
|
if (ckDiscoveringInDom()) ckStartDiscoveryPolling(0);
|
|
9
9
|
});
|
|
10
10
|
|
|
11
|
+
document.addEventListener("turbo:render", ckAutoFocusFirstError);
|
|
12
|
+
|
|
11
13
|
document.addEventListener("input", function(e) {
|
|
12
14
|
if (!e.target || e.target.id !== "tag_name") return;
|
|
13
15
|
var text = document.getElementById("tag-pill-text");
|
|
@@ -214,37 +216,56 @@ document.addEventListener("click", function(e) {
|
|
|
214
216
|
});
|
|
215
217
|
});
|
|
216
218
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
valid_json: [],
|
|
223
|
-
json_path_equals: ["json_path", "expected"],
|
|
224
|
-
length_bounds: ["min", "max"]
|
|
225
|
-
};
|
|
219
|
+
function ckCheckEditor(scope) {
|
|
220
|
+
if (!scope) return null;
|
|
221
|
+
if (scope.matches && scope.matches("[data-ck-check-form]")) return scope;
|
|
222
|
+
return scope.querySelector ? scope.querySelector("[data-ck-check-form]") : null;
|
|
223
|
+
}
|
|
226
224
|
|
|
227
|
-
|
|
225
|
+
function ckCheckForm(editor) {
|
|
226
|
+
try {
|
|
227
|
+
return JSON.parse(editor.getAttribute("data-ck-check-form"));
|
|
228
|
+
} catch (e) {
|
|
229
|
+
return null;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function ckApplyCheckLabels(editor, form, kind) {
|
|
234
|
+
var labels = form.labels[kind] || {};
|
|
235
|
+
var hints = form.hints[kind] || {};
|
|
236
|
+
editor.querySelectorAll("[data-ck-check-label]").forEach(function(node) {
|
|
237
|
+
var key = node.getAttribute("data-ck-check-label");
|
|
238
|
+
node.textContent = labels[key] || form.defaultLabels[key] || "";
|
|
239
|
+
});
|
|
240
|
+
editor.querySelectorAll("[data-ck-check-hint]").forEach(function(node) {
|
|
241
|
+
var key = node.getAttribute("data-ck-check-hint");
|
|
242
|
+
node.textContent = hints[key] || form.defaultHints[key] || "";
|
|
243
|
+
});
|
|
244
|
+
}
|
|
228
245
|
|
|
229
246
|
function ckApplyCheckFields(scope) {
|
|
230
|
-
|
|
231
|
-
|
|
247
|
+
var editor = ckCheckEditor(scope);
|
|
248
|
+
if (!editor) return;
|
|
249
|
+
var form = ckCheckForm(editor);
|
|
250
|
+
if (!form) return;
|
|
251
|
+
var kindSelect = editor.querySelector('[name="metric[check_config][check_kind]"]');
|
|
232
252
|
if (!kindSelect) return;
|
|
233
253
|
var kind = kindSelect.value;
|
|
234
|
-
var visible = (
|
|
235
|
-
var targetSelect =
|
|
254
|
+
var visible = (form.fields[kind] || []).slice();
|
|
255
|
+
var targetSelect = editor.querySelector('[name="metric[check_config][target]"]');
|
|
236
256
|
var targetIsJsonPath = !!(targetSelect && targetSelect.value === "json_path");
|
|
257
|
+
var expectedKey = form.expected[kind];
|
|
237
258
|
|
|
238
|
-
if (
|
|
259
|
+
if (expectedKey) {
|
|
239
260
|
visible.push("compare_to");
|
|
240
|
-
var comparison =
|
|
261
|
+
var comparison = editor.querySelector('[name="metric[check_config][compare_to]"]:checked');
|
|
241
262
|
if (comparison && comparison.value === "expected") {
|
|
242
|
-
visible = visible.filter(function(key) { return key !==
|
|
263
|
+
visible = visible.filter(function(key) { return key !== expectedKey; });
|
|
243
264
|
visible.push("expected_path");
|
|
244
265
|
}
|
|
245
266
|
}
|
|
246
267
|
|
|
247
|
-
|
|
268
|
+
editor.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
|
|
248
269
|
var key = field.getAttribute("data-ck-check-field");
|
|
249
270
|
var show;
|
|
250
271
|
if (key === "target_path") {
|
|
@@ -254,6 +275,8 @@ function ckApplyCheckFields(scope) {
|
|
|
254
275
|
}
|
|
255
276
|
field.hidden = !show;
|
|
256
277
|
});
|
|
278
|
+
|
|
279
|
+
ckApplyCheckLabels(editor, form, kind);
|
|
257
280
|
}
|
|
258
281
|
|
|
259
282
|
function ckApplyMetricType(group) {
|
|
@@ -271,14 +294,17 @@ function ckApplyMetricType(group) {
|
|
|
271
294
|
ckApplyCheckFields(scope);
|
|
272
295
|
}
|
|
273
296
|
|
|
274
|
-
|
|
297
|
+
function ckApplyMetricForms() {
|
|
275
298
|
document.querySelectorAll("[data-ck-metric-type]").forEach(function(group) {
|
|
276
299
|
ckApplyMetricType(group);
|
|
277
300
|
});
|
|
278
|
-
document.querySelectorAll(
|
|
301
|
+
document.querySelectorAll("[data-ck-check-form]").forEach(function(editor) {
|
|
279
302
|
ckApplyCheckFields(editor);
|
|
280
303
|
});
|
|
281
|
-
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
document.addEventListener("turbo:load", ckApplyMetricForms);
|
|
307
|
+
document.addEventListener("turbo:render", ckApplyMetricForms);
|
|
282
308
|
|
|
283
309
|
document.addEventListener("change", function(e) {
|
|
284
310
|
var target = e.target;
|
|
@@ -305,11 +331,14 @@ function ckApplyProviderFields(scope) {
|
|
|
305
331
|
});
|
|
306
332
|
}
|
|
307
333
|
|
|
308
|
-
|
|
334
|
+
function ckApplyProviderForms() {
|
|
309
335
|
document.querySelectorAll("[data-ck-provider-select]").forEach(function(select) {
|
|
310
336
|
ckApplyProviderFields(select.closest("form") || document);
|
|
311
337
|
});
|
|
312
|
-
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
document.addEventListener("turbo:load", ckApplyProviderForms);
|
|
341
|
+
document.addEventListener("turbo:render", ckApplyProviderForms);
|
|
313
342
|
|
|
314
343
|
document.addEventListener("change", function(e) {
|
|
315
344
|
var target = e.target;
|
|
@@ -2053,6 +2053,10 @@ label.ck-checkbox[hidden] {
|
|
|
2053
2053
|
flex: 1;
|
|
2054
2054
|
}
|
|
2055
2055
|
|
|
2056
|
+
.ck-field-row:not(:has(> .ck-field:not([hidden]))) {
|
|
2057
|
+
display: none;
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2056
2060
|
.ck-metric-type-chip {
|
|
2057
2061
|
text-transform: uppercase;
|
|
2058
2062
|
letter-spacing: 0.04em;
|
|
@@ -2060,32 +2064,42 @@ label.ck-checkbox[hidden] {
|
|
|
2060
2064
|
}
|
|
2061
2065
|
|
|
2062
2066
|
.ck-check-spec {
|
|
2063
|
-
display:
|
|
2064
|
-
|
|
2065
|
-
gap:
|
|
2067
|
+
display: grid;
|
|
2068
|
+
grid-template-columns: max-content minmax(0, 1fr);
|
|
2069
|
+
column-gap: 1rem;
|
|
2070
|
+
row-gap: 0.4rem;
|
|
2066
2071
|
margin: 0.5rem 0 0;
|
|
2067
2072
|
}
|
|
2068
2073
|
|
|
2069
2074
|
.ck-check-spec__row {
|
|
2070
|
-
display:
|
|
2071
|
-
|
|
2075
|
+
display: grid;
|
|
2076
|
+
grid-column: 1 / -1;
|
|
2077
|
+
grid-template-columns: subgrid;
|
|
2072
2078
|
align-items: baseline;
|
|
2073
2079
|
}
|
|
2074
2080
|
|
|
2075
2081
|
.ck-check-spec__term {
|
|
2076
|
-
flex: 0 0 7rem;
|
|
2077
2082
|
margin: 0;
|
|
2078
2083
|
font-size: 0.8rem;
|
|
2079
2084
|
text-transform: uppercase;
|
|
2080
2085
|
letter-spacing: 0.04em;
|
|
2086
|
+
white-space: nowrap;
|
|
2081
2087
|
color: var(--ck-muted);
|
|
2082
2088
|
}
|
|
2083
2089
|
|
|
2084
2090
|
.ck-check-spec__val {
|
|
2085
2091
|
margin: 0;
|
|
2092
|
+
min-width: 0;
|
|
2093
|
+
overflow-wrap: anywhere;
|
|
2086
2094
|
color: var(--ck-text);
|
|
2087
2095
|
}
|
|
2088
2096
|
|
|
2097
|
+
@media (max-width: 640px) {
|
|
2098
|
+
.ck-check-spec { grid-template-columns: minmax(0, 1fr); row-gap: 0.6rem; }
|
|
2099
|
+
.ck-check-spec__row { grid-template-columns: minmax(0, 1fr); row-gap: 0.15rem; }
|
|
2100
|
+
.ck-check-spec__term { white-space: normal; }
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2089
2103
|
.ck-check-spec__code {
|
|
2090
2104
|
font-family: var(--ck-mono);
|
|
2091
2105
|
font-size: 0.85rem;
|
|
@@ -2098,6 +2112,7 @@ label.ck-checkbox[hidden] {
|
|
|
2098
2112
|
|
|
2099
2113
|
.ck-select {
|
|
2100
2114
|
position: relative;
|
|
2115
|
+
min-width: 0;
|
|
2101
2116
|
}
|
|
2102
2117
|
|
|
2103
2118
|
.ck-select__trigger {
|
|
@@ -2311,6 +2326,15 @@ label.ck-checkbox[hidden] {
|
|
|
2311
2326
|
flex: 1;
|
|
2312
2327
|
}
|
|
2313
2328
|
|
|
2329
|
+
.ck-select-with-action > .field_with_errors {
|
|
2330
|
+
flex: 1;
|
|
2331
|
+
min-width: 0;
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2334
|
+
.ck-select-with-action > .field_with_errors > .ck-input {
|
|
2335
|
+
width: 100%;
|
|
2336
|
+
}
|
|
2337
|
+
|
|
2314
2338
|
.ck-icon-btn:hover {
|
|
2315
2339
|
background: rgba(255, 255, 255, 0.05);
|
|
2316
2340
|
border-color: var(--ck-line-strong);
|
|
@@ -3262,13 +3286,13 @@ select.ck-input {
|
|
|
3262
3286
|
|
|
3263
3287
|
.ck-run-compare-table td { vertical-align: top; }
|
|
3264
3288
|
|
|
3265
|
-
.ck-table-scroll { overflow-x: auto; }
|
|
3289
|
+
.ck-table-scroll { position: relative; overflow-x: auto; }
|
|
3266
3290
|
|
|
3267
3291
|
.ck-run-compare-table th { white-space: nowrap; }
|
|
3268
3292
|
.ck-run-compare-table th:nth-child(1) { width: 7rem; }
|
|
3269
|
-
.ck-run-compare-table th:nth-child(2) { width:
|
|
3270
|
-
.ck-run-compare-table th:nth-child(3) { width:
|
|
3271
|
-
.ck-run-compare-table th:nth-child(4) { width:
|
|
3293
|
+
.ck-run-compare-table th:nth-child(2) { width: 12rem; }
|
|
3294
|
+
.ck-run-compare-table th:nth-child(3) { width: 16rem; text-align: center; }
|
|
3295
|
+
.ck-run-compare-table th:nth-child(4) { width: 9rem; }
|
|
3272
3296
|
.ck-run-compare-table th:nth-child(5) { width: auto; }
|
|
3273
3297
|
|
|
3274
3298
|
.ck-run-compare-table th.ck-num,
|
|
@@ -3283,13 +3307,26 @@ select.ck-input {
|
|
|
3283
3307
|
font-weight: 500;
|
|
3284
3308
|
}
|
|
3285
3309
|
|
|
3286
|
-
.ck-cmp-score__pair {
|
|
3310
|
+
.ck-cmp-score__pair {
|
|
3311
|
+
display: grid;
|
|
3312
|
+
grid-template-columns: minmax(max-content, 1fr) auto minmax(max-content, 1fr);
|
|
3313
|
+
align-items: center;
|
|
3314
|
+
gap: 0.45rem;
|
|
3315
|
+
width: 100%;
|
|
3316
|
+
}
|
|
3317
|
+
.ck-cmp-score__pair > :first-child { justify-self: end; }
|
|
3318
|
+
.ck-cmp-score__pair > :last-child { justify-self: start; }
|
|
3319
|
+
@media (max-width: 640px) {
|
|
3320
|
+
.ck-cmp-score__pair { display: inline-grid; width: auto; }
|
|
3321
|
+
}
|
|
3322
|
+
.ck-cmp-score__pair .ck-badge { white-space: nowrap; }
|
|
3287
3323
|
.ck-cmp-score__arrow { color: var(--ck-dim); font-size: 0.78rem; }
|
|
3288
3324
|
.ck-cmp-score__link { display: inline-flex; text-decoration: none; border-radius: 7px; transition: box-shadow 0.12s ease; }
|
|
3289
3325
|
.ck-cmp-score__link:hover { box-shadow: 0 0 0 1px var(--ck-accent); }
|
|
3290
3326
|
.ck-cmp-score__none { color: var(--ck-dim); }
|
|
3291
3327
|
|
|
3292
|
-
.ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; }
|
|
3328
|
+
.ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; white-space: nowrap; }
|
|
3329
|
+
.ck-cmp-delta__pair { display: inline-flex; align-items: baseline; justify-content: flex-end; gap: 0.35rem; }
|
|
3293
3330
|
.ck-delta__arrow { font-size: 0.72rem; }
|
|
3294
3331
|
|
|
3295
3332
|
.ck-cmp-ver__one, .ck-cmp-ver__pair {
|
|
@@ -5175,11 +5212,15 @@ a.tag-mark {
|
|
|
5175
5212
|
}
|
|
5176
5213
|
|
|
5177
5214
|
|
|
5178
|
-
.ck-input--error
|
|
5215
|
+
.ck-input--error,
|
|
5216
|
+
.ck-input[aria-invalid="true"],
|
|
5217
|
+
.ck-select__trigger[aria-invalid="true"] {
|
|
5179
5218
|
border-color: var(--ck-danger);
|
|
5180
|
-
background: rgba(248, 113, 113, 0.05);
|
|
5219
|
+
background-color: rgba(248, 113, 113, 0.05);
|
|
5181
5220
|
}
|
|
5182
|
-
.ck-input--error:focus
|
|
5221
|
+
.ck-input--error:focus,
|
|
5222
|
+
.ck-input[aria-invalid="true"]:focus,
|
|
5223
|
+
.ck-select__trigger[aria-invalid="true"]:focus-visible {
|
|
5183
5224
|
box-shadow: 0 0 0 3px rgba(248, 113, 113, 0.15);
|
|
5184
5225
|
border-color: var(--ck-danger);
|
|
5185
5226
|
}
|
|
@@ -70,7 +70,7 @@ module CompletionKit
|
|
|
70
70
|
def metric_params
|
|
71
71
|
params.permit(:name, :instruction, :metric_type,
|
|
72
72
|
rubric_bands: [:stars, :description],
|
|
73
|
-
check_config:
|
|
73
|
+
check_config: [*%i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim], { value: [] }],
|
|
74
74
|
tag_names: [])
|
|
75
75
|
end
|
|
76
76
|
end
|
|
@@ -112,9 +112,12 @@ module CompletionKit
|
|
|
112
112
|
right_score: r_review ? r_review.ai_score : nil,
|
|
113
113
|
left_passed: l_review&.passed,
|
|
114
114
|
right_passed: r_review&.passed,
|
|
115
|
+
left_check_score: l_review&.check_score&.to_f,
|
|
116
|
+
right_check_score: r_review&.check_score&.to_f,
|
|
115
117
|
left_metric_version_id: l_review&.metric_version_id,
|
|
116
118
|
right_metric_version_id: r_review&.metric_version_id,
|
|
117
|
-
delta: (l_review&.ai_score
|
|
119
|
+
delta: CompletionKit::RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
|
|
120
|
+
check_score_delta: CompletionKit::RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
|
|
118
121
|
result_change: CompletionKit::RunComparison.result_change(l_review&.passed, r_review&.passed)
|
|
119
122
|
}
|
|
120
123
|
end.compact
|
|
@@ -61,7 +61,9 @@ module CompletionKit
|
|
|
61
61
|
@published_metric_version = MetricVersion.published.where(metric_id: @metric.id, current: true).first
|
|
62
62
|
@improve_disagreement_count = @metric.check? ? 0 : Agreement.where(metric_id: @metric.id, verdict: "disagree", run_id: CompletionKit::Run.visible_run_ids).count
|
|
63
63
|
|
|
64
|
-
if @edit_draft
|
|
64
|
+
if @edit_draft&.check?
|
|
65
|
+
@metric.check_config = @edit_draft.check_config
|
|
66
|
+
elsif @edit_draft
|
|
65
67
|
@metric.instruction = @edit_draft.instruction
|
|
66
68
|
@metric.rubric_bands = @edit_draft.rubric_bands
|
|
67
69
|
end
|
|
@@ -78,17 +80,12 @@ module CompletionKit
|
|
|
78
80
|
end
|
|
79
81
|
|
|
80
82
|
def update
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
end
|
|
83
|
+
attrs = metric_params
|
|
84
|
+
@metric.assign_attributes(attrs.except(:instruction, :rubric_bands, :check_config))
|
|
85
|
+
return update_check_definition(attrs.to_unsafe_h[:check_config]) if @metric.check?
|
|
86
|
+
return render(:edit, status: :unprocessable_entity) unless @metric.save
|
|
86
87
|
|
|
87
|
-
|
|
88
|
-
update_check_definition
|
|
89
|
-
else
|
|
90
|
-
update_judge_definition
|
|
91
|
-
end
|
|
88
|
+
update_judge_definition
|
|
92
89
|
end
|
|
93
90
|
|
|
94
91
|
def destroy
|
|
@@ -157,7 +154,11 @@ module CompletionKit
|
|
|
157
154
|
reverting = was_published_already && !version.current?
|
|
158
155
|
previously_current = MetricVersion.current.find_by(metric_id: @metric.id)
|
|
159
156
|
|
|
160
|
-
|
|
157
|
+
begin
|
|
158
|
+
version.publish!
|
|
159
|
+
rescue ActiveRecord::RecordInvalid => e
|
|
160
|
+
return redirect_to(metric_path(@metric), alert: "#{version.version_label} was not published. #{e.record.errors.full_messages.to_sentence}.")
|
|
161
|
+
end
|
|
161
162
|
|
|
162
163
|
if reverting
|
|
163
164
|
redirect_to metric_path(@metric),
|
|
@@ -207,28 +208,28 @@ module CompletionKit
|
|
|
207
208
|
end
|
|
208
209
|
end
|
|
209
210
|
|
|
210
|
-
def update_check_definition
|
|
211
|
-
|
|
212
|
-
|
|
211
|
+
def update_check_definition(proposed)
|
|
212
|
+
live = @metric.check_config
|
|
213
|
+
changed = !proposed.nil? && CompletionKit::Checks::Registry.canonical(proposed) != CompletionKit::Checks::Registry.canonical(live)
|
|
214
|
+
@metric.check_config = proposed if changed
|
|
215
|
+
return render(:edit, status: :unprocessable_entity) unless @metric.valid?
|
|
216
|
+
return save_check_draft(live, proposed) if changed && @metric.reviews.exists?
|
|
213
217
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
218
|
+
@metric.save!
|
|
219
|
+
MetricVersion.published.find_by(metric_id: @metric.id, current: true)&.update!(metric_type: "check", check_config: proposed) if changed
|
|
220
|
+
redirect_to metric_path(@metric), notice: "Metric was successfully updated."
|
|
221
|
+
end
|
|
217
222
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
|
|
229
|
-
current_pub&.update!(metric_type: "check", check_config: proposed)
|
|
230
|
-
redirect_to metric_path(@metric), notice: "Metric was successfully updated."
|
|
231
|
-
end
|
|
223
|
+
def save_check_draft(live, proposed)
|
|
224
|
+
@metric.check_config = live
|
|
225
|
+
@metric.save!(validate: false)
|
|
226
|
+
MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
|
|
227
|
+
draft = MetricVersion.create!(
|
|
228
|
+
metric: @metric, metric_type: "check", check_config: proposed,
|
|
229
|
+
state: "draft", source: "edit", current: false
|
|
230
|
+
)
|
|
231
|
+
redirect_to edit_metric_path(@metric),
|
|
232
|
+
notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
|
|
232
233
|
end
|
|
233
234
|
|
|
234
235
|
def set_metric
|
|
@@ -247,7 +248,7 @@ module CompletionKit
|
|
|
247
248
|
def metric_params
|
|
248
249
|
permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
|
|
249
250
|
rubric_bands: [:stars, :description],
|
|
250
|
-
check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
|
|
251
|
+
check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim],
|
|
251
252
|
tag_names: [])
|
|
252
253
|
permitted[:check_config] = normalize_check_config(permitted[:check_config]) if permitted.key?(:check_config)
|
|
253
254
|
permitted
|
|
@@ -255,22 +256,23 @@ module CompletionKit
|
|
|
255
256
|
|
|
256
257
|
def normalize_check_config(config)
|
|
257
258
|
hash = config.to_unsafe_h.stringify_keys
|
|
258
|
-
%w[min max].each { |key| hash[key] = hash[key]
|
|
259
|
+
%w[min max tolerance].each { |key| hash[key] = coerce_number(hash[key]) if hash[key].present? }
|
|
259
260
|
%w[case_sensitive multiline trim].each { |key| hash[key] = ActiveModel::Type::Boolean.new.cast(hash[key]) if hash.key?(key) }
|
|
260
|
-
hash["expected"] =
|
|
261
|
-
|
|
262
|
-
hash.delete("value")
|
|
263
|
-
else
|
|
264
|
-
hash.delete("compare_to")
|
|
265
|
-
hash.delete("expected_path")
|
|
266
|
-
end
|
|
267
|
-
hash.reject { |_, value| value.nil? || value == "" }
|
|
261
|
+
hash["expected"] = CompletionKit::Checks::Registry.parse_json(hash["expected"]) if hash["expected"].present?
|
|
262
|
+
prune_check_config(hash.except(*CompletionKit::Checks::Registry.unused_keys(hash)))
|
|
268
263
|
end
|
|
269
264
|
|
|
270
|
-
def
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
value
|
|
265
|
+
def prune_check_config(hash)
|
|
266
|
+
allowed = CompletionKit::Checks::Registry::SHARED_KEYS +
|
|
267
|
+
CompletionKit::Checks::Registry.config_keys(hash["check_kind"])
|
|
268
|
+
hash.slice(*allowed).reject { |_, value| value.nil? || value == "" }
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def coerce_number(value)
|
|
272
|
+
number = CompletionKit::Checks::NumericValue.parse(value)
|
|
273
|
+
return value if number.nil?
|
|
274
|
+
|
|
275
|
+
number == number.to_i ? number.to_i : number
|
|
274
276
|
end
|
|
275
277
|
|
|
276
278
|
def normalize_rubric_bands_for_update(bands)
|
|
@@ -21,7 +21,7 @@ module CompletionKit
|
|
|
21
21
|
|
|
22
22
|
def sample_data
|
|
23
23
|
Onboarding::SampleData.install!
|
|
24
|
-
redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt
|
|
24
|
+
redirect_to onboarding_path, notice: "Loaded a sample dataset and prompt. Edit or delete them whenever."
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
end
|
|
@@ -40,6 +40,7 @@ module CompletionKit
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
def create
|
|
43
|
+
@judge_only = judge_only_chosen?
|
|
43
44
|
@run = Run.new(run_attributes)
|
|
44
45
|
if @run.save
|
|
45
46
|
redirect_to run_path(@run), notice: "Run was successfully created."
|
|
@@ -50,10 +51,11 @@ module CompletionKit
|
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
def update
|
|
54
|
+
@judge_only = judge_only_chosen?
|
|
53
55
|
if @run.responses.any? && run_generation_changed?
|
|
54
56
|
attrs = run_attributes.to_h
|
|
55
57
|
attrs.delete("name") if attrs["name"].to_s == @run.name.to_s
|
|
56
|
-
new_run = Run.new(attrs.merge(status: "pending"))
|
|
58
|
+
new_run = Run.new(attrs.merge(status: "pending", cloned_from_judge_model: @run.judge_model))
|
|
57
59
|
if new_run.save
|
|
58
60
|
redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
|
|
59
61
|
else
|
|
@@ -200,9 +202,12 @@ module CompletionKit
|
|
|
200
202
|
right_score: r_review ? r_review.ai_score : nil,
|
|
201
203
|
left_passed: l_review&.passed,
|
|
202
204
|
right_passed: r_review&.passed,
|
|
205
|
+
left_check_score: l_review&.check_score&.to_f,
|
|
206
|
+
right_check_score: r_review&.check_score&.to_f,
|
|
203
207
|
left_version_label: version_label_for(l_review, metric_versions),
|
|
204
208
|
right_version_label: version_label_for(r_review, metric_versions),
|
|
205
|
-
delta: (l_review&.ai_score
|
|
209
|
+
delta: RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
|
|
210
|
+
check_score_delta: RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
|
|
206
211
|
result_change: RunComparison.result_change(l_review&.passed, r_review&.passed)
|
|
207
212
|
}
|
|
208
213
|
end.compact
|
|
@@ -224,11 +229,16 @@ module CompletionKit
|
|
|
224
229
|
end
|
|
225
230
|
|
|
226
231
|
def run_params
|
|
227
|
-
params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
|
|
232
|
+
params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :max_tokens, :judge_temperature, :output_column, :expected_column, :judge_only, metric_ids: [], tag_names: [])
|
|
228
233
|
end
|
|
229
234
|
|
|
230
235
|
def run_attributes
|
|
231
|
-
run_params.except(:metric_ids).merge(selected_metric_ids: run_params[:metric_ids])
|
|
236
|
+
attributes = run_params.except(:metric_ids, :judge_only).merge(selected_metric_ids: run_params[:metric_ids])
|
|
237
|
+
judge_only_chosen? ? attributes.merge(prompt_id: nil) : attributes
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def judge_only_chosen?
|
|
241
|
+
run_params[:judge_only] == "1"
|
|
232
242
|
end
|
|
233
243
|
|
|
234
244
|
# Editing a run that already has results forks a new run — but only when a
|
|
@@ -237,9 +247,10 @@ module CompletionKit
|
|
|
237
247
|
GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature max_tokens judge_temperature output_column expected_column].freeze
|
|
238
248
|
|
|
239
249
|
def run_generation_changed?
|
|
250
|
+
attributes = run_attributes
|
|
240
251
|
GENERATION_RUN_FIELDS.each do |field|
|
|
241
|
-
next unless
|
|
242
|
-
return true if normalize_run_field(field,
|
|
252
|
+
next unless attributes.key?(field)
|
|
253
|
+
return true if normalize_run_field(field, attributes[field]) != normalize_run_field(field, @run.public_send(field))
|
|
243
254
|
end
|
|
244
255
|
return false unless params[:run].key?(:metric_ids)
|
|
245
256
|
Array(params[:run][:metric_ids]).map(&:to_i).reject(&:zero?).sort != @run.metric_ids.sort
|
|
@@ -247,6 +258,7 @@ module CompletionKit
|
|
|
247
258
|
|
|
248
259
|
def normalize_run_field(field, value)
|
|
249
260
|
s = value.to_s.strip
|
|
261
|
+
return s.presence || "actual_output" if field == :output_column
|
|
250
262
|
return nil if s.empty?
|
|
251
263
|
case field
|
|
252
264
|
when :temperature, :judge_temperature then s.to_f
|