completion-kit 0.28.45 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/completion_kit/application.js +52 -23
- data/app/assets/stylesheets/completion_kit/application.css +56 -15
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
- data/app/controllers/completion_kit/runs_controller.rb +18 -6
- data/app/helpers/completion_kit/application_helper.rb +110 -52
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +1 -1
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +36 -18
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +2 -2
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
- data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
- data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
- data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/onboarding/show.html.erb +2 -2
- data/app/views/completion_kit/prompts/_form.html.erb +2 -1
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +21 -11
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
- data/app/views/completion_kit/runs/_row.html.erb +10 -8
- data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/edit.html.erb +1 -1
- data/app/views/completion_kit/runs/new.html.erb +1 -1
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +1 -1
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
- data/lib/completion_kit/errors.rb +2 -2
- data/lib/completion_kit/version.rb +1 -1
- metadata +10 -1
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
<div class="ck-field">
|
|
21
21
|
<label class="ck-checkbox-label">
|
|
22
|
-
<%= check_box_tag "run[judge_only]", "1", run.persisted? && run.judge_only
|
|
22
|
+
<%= check_box_tag "run[judge_only]", "1", local_assigns[:judge_only] || (run.persisted? && run.judge_only?), id: "run_judge_only", class: "ck-checkbox" %>
|
|
23
23
|
<span class="ck-checkbox-label__box" aria-hidden="true"></span>
|
|
24
24
|
<span class="ck-checkbox-label__body">
|
|
25
25
|
<span class="ck-checkbox-label__text">Score existing outputs</span>
|
|
@@ -98,10 +98,11 @@
|
|
|
98
98
|
<% available = CompletionKit::ApiConfig.available_models(scope: :judging) %>
|
|
99
99
|
<% if available.any? %>
|
|
100
100
|
<div class="ck-select-with-action">
|
|
101
|
-
<%= form.select :judge_model, ck_grouped_models(available, run.judge_model), { include_blank: "None" }, { class: "ck-input", id: "run_judge_model" } %>
|
|
101
|
+
<%= form.select :judge_model, ck_grouped_models(available, run.judge_model, rejected: run.errors[:judge_model].any?), { include_blank: "None" }, { class: "ck-input", id: "run_judge_model", **ck_field_aria(form, :judge_model) } %>
|
|
102
102
|
<% ck_refreshing = CompletionKit::ProviderCredential.discovery_in_progress? %>
|
|
103
103
|
<button type="button" class="ck-icon-btn<%= ' ck-icon-btn--spinning' if ck_refreshing %>" title="Refresh models" <%= 'disabled' if ck_refreshing %> onclick="ckRefreshModels()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" fill="currentColor" width="16" height="16"><path fill-rule="evenodd" d="M13.836 2.477a.75.75 0 0 1 .75.75v3.182a.75.75 0 0 1-.75.75h-3.182a.75.75 0 0 1 0-1.5h1.37l-.84-.841a4.5 4.5 0 0 0-7.08.681.75.75 0 0 1-1.264-.808 6 6 0 0 1 9.44-.908l.84.84V3.227a.75.75 0 0 1 .75-.75Zm-.911 7.5A.75.75 0 0 1 13.199 11a6 6 0 0 1-9.44.908l-.84-.84v1.68a.75.75 0 0 1-1.5 0V9.567a.75.75 0 0 1 .75-.75h3.182a.75.75 0 0 1 0 1.5h-1.37l.84.841a4.5 4.5 0 0 0 7.08-.681.75.75 0 0 1 1.024-.274Z" clip-rule="evenodd"/></svg></button>
|
|
104
104
|
</div>
|
|
105
|
+
<%= ck_field_error(form, :judge_model) %>
|
|
105
106
|
<p class="ck-field-hint" id="judge-hint"></p>
|
|
106
107
|
<div hidden data-refresh-progress-carriers data-ck-refresh-url="<%= refresh_models_path %>" data-ck-statuses-url="<%= statuses_provider_credentials_path %>">
|
|
107
108
|
<% CompletionKit::ProviderCredential.find_each do |pc| %>
|
|
@@ -112,7 +113,7 @@
|
|
|
112
113
|
<p class="ck-field-hint" id="refresh-status" role="status" aria-live="polite" style="min-height: 1.2em; margin-top: -0.25rem; font-size: 0.75rem;"> </p>
|
|
113
114
|
<% else %>
|
|
114
115
|
<p class="ck-field-hint" style="color: var(--ck-warning);">No judge models available. <%= link_to "Add a provider", provider_credentials_path, class: "ck-link" %></p>
|
|
115
|
-
<%= form.hidden_field :judge_model,
|
|
116
|
+
<%= form.hidden_field :judge_model, id: "run_judge_model" %>
|
|
116
117
|
<% end %>
|
|
117
118
|
</div>
|
|
118
119
|
|
|
@@ -157,9 +158,10 @@
|
|
|
157
158
|
</div>
|
|
158
159
|
<% end %>
|
|
159
160
|
|
|
161
|
+
<%= hidden_field_tag "run[metric_ids][]", "", id: nil %>
|
|
160
162
|
<div class="ck-metric-checkboxes" data-metric-checkboxes role="group" aria-labelledby="run_metrics_label">
|
|
161
163
|
<% @all_metrics.each do |metric| %>
|
|
162
|
-
<label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>" data-compare-expected="<%= metric.check? && metric.check_config.to_h["compare_to"] == "expected" ? "1" : "0" %>">
|
|
164
|
+
<label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>" data-needs-judge="<%= metric.llm_judge? ? "1" : "0" %>" data-compare-expected="<%= metric.check? && metric.check_config.to_h["compare_to"] == "expected" ? "1" : "0" %>">
|
|
163
165
|
<%= check_box_tag "run[metric_ids][]", metric.id, run.selected_metric_ids.include?(metric.id), class: "ck-checkbox", id: "run_metric_#{metric.id}" %>
|
|
164
166
|
<span class="ck-checkbox-label__box" aria-hidden="true"></span>
|
|
165
167
|
<span class="ck-checkbox-label__body">
|
|
@@ -194,6 +196,7 @@ function updateRunForm() {
|
|
|
194
196
|
|
|
195
197
|
var prompt = promptEl ? promptEl.value : '';
|
|
196
198
|
var judge = judgeEl ? judgeEl.value : '';
|
|
199
|
+
var judgeListed = !!judgeEl && judgeEl.type !== 'hidden';
|
|
197
200
|
var metrics = document.querySelectorAll('input[name="run[metric_ids][]"]:checked');
|
|
198
201
|
var judgeField = document.getElementById('judge-field');
|
|
199
202
|
var metricsField = document.getElementById('metrics-field');
|
|
@@ -206,13 +209,21 @@ function updateRunForm() {
|
|
|
206
209
|
if (judgeHint) judgeHint.textContent = '';
|
|
207
210
|
if (metricsHint) metricsHint.textContent = '';
|
|
208
211
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
+
var judgedMetrics = Array.prototype.filter.call(metrics, function(cb) {
|
|
213
|
+
var lbl = cb.closest('[data-needs-judge]');
|
|
214
|
+
return !!lbl && lbl.dataset.needsJudge === '1';
|
|
215
|
+
});
|
|
216
|
+
var judgeUnpaired = judgeListed && !!judge && metrics.length === 0;
|
|
217
|
+
var metricsUnpaired = judgeListed && !judge && judgedMetrics.length > 0;
|
|
218
|
+
if (judgeListed && judge && metrics.length > 0 && judgedMetrics.length === 0) {
|
|
219
|
+
if (judgeHint) judgeHint.textContent = 'Checks need no judge, so this run will not use it.';
|
|
220
|
+
} else if (judgeListed && judge && metrics.length > 0) {
|
|
221
|
+
if (judgeHint) judgeHint.textContent = judgeOnly ? 'Existing outputs will be judged automatically.' : 'Responses will be generated then judged automatically.';
|
|
222
|
+
} else if (judgeUnpaired) {
|
|
212
223
|
if (judgeField) judgeField.className = 'ck-field ck-field--info';
|
|
213
224
|
if (metricsField) metricsField.className = 'ck-field ck-field--info';
|
|
214
225
|
if (metricsHint) metricsHint.textContent = 'Select at least one metric or group to enable judging.';
|
|
215
|
-
} else if (
|
|
226
|
+
} else if (metricsUnpaired) {
|
|
216
227
|
if (judgeField) judgeField.className = 'ck-field ck-field--info';
|
|
217
228
|
if (judgeHint) judgeHint.textContent = 'Select a judge model to enable judging.';
|
|
218
229
|
}
|
|
@@ -286,7 +297,7 @@ function updateRunForm() {
|
|
|
286
297
|
if (col === '' || headersJudge.indexOf(col) === -1) {
|
|
287
298
|
valid = false;
|
|
288
299
|
if (datasetField) datasetField.className = 'ck-field ck-field--error';
|
|
289
|
-
if (datasetHint) datasetHint.textContent = "Dataset has no \"" + col + "\" column
|
|
300
|
+
if (datasetHint) datasetHint.textContent = "Dataset has no \"" + col + "\" column. Pick a different output column or dataset.";
|
|
290
301
|
}
|
|
291
302
|
} else if (!dataset) {
|
|
292
303
|
if (datasetField) datasetField.className = 'ck-field ck-field--info';
|
|
@@ -297,8 +308,7 @@ function updateRunForm() {
|
|
|
297
308
|
if (hasVars && !dataset) valid = false;
|
|
298
309
|
if (missingVars.length > 0) valid = false;
|
|
299
310
|
}
|
|
300
|
-
if (
|
|
301
|
-
if (!judge && metrics.length > 0) valid = false;
|
|
311
|
+
if (judgeUnpaired || metricsUnpaired) valid = false;
|
|
302
312
|
if (submitBtn) submitBtn.disabled = !valid;
|
|
303
313
|
|
|
304
314
|
ckUpdateMetricGroupsState();
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
<td class="ck-response-cell__text">
|
|
5
5
|
<% if response.status == "failed" %>
|
|
6
6
|
<% err = response.error_payload %>
|
|
7
|
-
|
|
7
|
+
<% err_prefix = [err && err[:provider]&.titleize, err && err[:status]].compact.join(" ") %>
|
|
8
|
+
<span class="ck-response-cell__preview ck-response-cell__error"><%= "#{err_prefix}: " if err_prefix.present? %><%= truncate(err && err[:message].to_s, length: 600) %></span>
|
|
8
9
|
<% else %>
|
|
9
10
|
<span class="ck-response-cell__preview"><%= truncate(response.response_text.to_s, length: 600) %></span>
|
|
10
11
|
<% end %>
|
|
@@ -20,9 +21,10 @@
|
|
|
20
21
|
<span class="ck-metric-pip__label" aria-hidden="true"><%= r.metric_name %> <strong><%= r.ai_score %></strong></span>
|
|
21
22
|
</span>
|
|
22
23
|
<% else %>
|
|
23
|
-
|
|
24
|
+
<% verdict = ck_check_verdict(r.passed, r.check_score) %>
|
|
25
|
+
<span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(r.passed ? 1.0 : 0.0) %>" role="img" aria-label="<%= r.metric_name %> <%= verdict %>">
|
|
24
26
|
<span class="ck-metric-pip__bar"></span>
|
|
25
|
-
<span class="ck-metric-pip__label" aria-hidden="true"><%= r.metric_name %> <strong><%=
|
|
27
|
+
<span class="ck-metric-pip__label" aria-hidden="true"><%= r.metric_name %> <strong><%= verdict %></strong></span>
|
|
26
28
|
</span>
|
|
27
29
|
<% end %>
|
|
28
30
|
<% end %>
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
<div class="ck-metric-bar ck-metric-bar--compact">
|
|
37
37
|
<% metrics.each do |m| %>
|
|
38
38
|
<% if m[:kind] == "check" %>
|
|
39
|
-
<span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(m[:pass_rate]) %>" role="img" aria-label="<%= m[:name]
|
|
39
|
+
<span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(m[:pass_rate]) %>" role="img" aria-label="<%= ck_check_average_label(m[:name], m) %>">
|
|
40
40
|
<span class="ck-metric-pip__bar"></span>
|
|
41
|
-
<span class="ck-metric-pip__label" aria-hidden="true"><%= m[:name] %> <strong><%= (m
|
|
41
|
+
<span class="ck-metric-pip__label" aria-hidden="true"><%= m[:name] %> <strong><%= ck_check_average(m) %></strong></span>
|
|
42
42
|
</span>
|
|
43
43
|
<% else %>
|
|
44
44
|
<span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(m[:avg]) %>" role="img" aria-label="<%= m[:name] %> average score <%= m[:avg] %>">
|
|
@@ -56,12 +56,14 @@
|
|
|
56
56
|
<% avg = run.avg_score %>
|
|
57
57
|
<% pass_rate = run.check_pass_rate %>
|
|
58
58
|
<% if avg || pass_rate %>
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
<div class="ck-chip-row">
|
|
60
|
+
<% if avg %>
|
|
61
|
+
<span class="<%= ck_badge_classes(ck_score_kind(avg)) %>"><%= avg %></span>
|
|
62
|
+
<% end %>
|
|
63
|
+
<% if pass_rate %>
|
|
64
|
+
<span class="<%= ck_badge_classes(ck_pass_rate_kind(pass_rate)) %>"><%= (pass_rate * 100).round %>%</span>
|
|
65
|
+
<% end %>
|
|
66
|
+
</div>
|
|
65
67
|
<% else %>
|
|
66
68
|
<span class="ck-runs-table__dim">—</span>
|
|
67
69
|
<% end %>
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
<div class="ck-run-config__row">
|
|
20
20
|
<span class="ck-run-config__key">Judge</span>
|
|
21
21
|
<span style="text-transform: none;"><%= run.judge_model %></span>
|
|
22
|
-
<%
|
|
22
|
+
<% if run.llm_metrics.any? && !run.llm_judge_configured? %>
|
|
23
23
|
<span class="ck-run-config__warn">provider not configured</span>
|
|
24
24
|
<% end %>
|
|
25
25
|
</div>
|
|
@@ -41,9 +41,9 @@
|
|
|
41
41
|
<% run.metrics.order(:name).each do |metric| %>
|
|
42
42
|
<% avg_for_metric = metric_lookup[metric.name.to_s.downcase] %>
|
|
43
43
|
<% if avg_for_metric && avg_for_metric[:kind] == "check" %>
|
|
44
|
-
<span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(avg_for_metric[:pass_rate]) %>">
|
|
44
|
+
<span class="ck-metric-pip ck-metric-pip--<%= ck_pass_rate_kind(avg_for_metric[:pass_rate]) %>" title="<%= ck_check_average_label(metric.name, avg_for_metric) %>">
|
|
45
45
|
<span class="ck-metric-pip__bar"></span>
|
|
46
|
-
<span class="ck-metric-pip__label"><%= metric.name %> <strong><%= (avg_for_metric
|
|
46
|
+
<span class="ck-metric-pip__label"><%= metric.name %> <strong><%= ck_check_average(avg_for_metric) %></strong></span>
|
|
47
47
|
</span>
|
|
48
48
|
<% elsif avg_for_metric %>
|
|
49
49
|
<span class="ck-metric-pip ck-metric-pip--<%= ck_score_kind(avg_for_metric[:avg]) %>">
|
|
@@ -56,18 +56,22 @@
|
|
|
56
56
|
<td class="ck-compare-metric" data-label="Metric" title="<%= pm[:metric_name] %>"><%= pm[:metric_name] %></td>
|
|
57
57
|
<td class="ck-cmp-score" data-label="A → B">
|
|
58
58
|
<span class="ck-cmp-score__pair">
|
|
59
|
-
<%= render "compare_score", kind: pm[:kind], passed: pm[:left_passed], score: pm[:left_score], response: row[:left_response], run: @run, side: "A" %>
|
|
59
|
+
<%= render "compare_score", kind: pm[:kind], passed: pm[:left_passed], score: pm[:left_score], check_score: pm[:left_check_score], response: row[:left_response], run: @run, side: "A" %>
|
|
60
60
|
<span class="ck-cmp-score__arrow" aria-hidden="true">→</span>
|
|
61
|
-
<%= render "compare_score", kind: pm[:kind], passed: pm[:right_passed], score: pm[:right_score], response: row[:right_response], run: @other_run, side: "B" %>
|
|
61
|
+
<%= render "compare_score", kind: pm[:kind], passed: pm[:right_passed], score: pm[:right_score], check_score: pm[:right_check_score], response: row[:right_response], run: @other_run, side: "B" %>
|
|
62
62
|
</span>
|
|
63
63
|
</td>
|
|
64
64
|
<td class="ck-num ck-cmp-delta" data-label="Δ">
|
|
65
65
|
<% if pm[:kind] == "check" %>
|
|
66
|
-
|
|
66
|
+
<% points = ck_check_score_change(pm[:left_check_score], pm[:right_check_score]) %>
|
|
67
|
+
<% badge = ck_result_change_badge(pm[:result_change]) unless points && pm[:result_change] == "same" %>
|
|
68
|
+
<% if badge || points %>
|
|
69
|
+
<span class="ck-cmp-delta__pair"><%= badge %><%= ck_delta_chip(points, "%") if points %></span>
|
|
70
|
+
<% else %>
|
|
71
|
+
<span class="ck-cmp-score__none">—</span>
|
|
72
|
+
<% end %>
|
|
67
73
|
<% elsif pm[:delta] %>
|
|
68
|
-
|
|
69
|
-
<% dcls = d > 0 ? "ck-delta--positive" : d < 0 ? "ck-delta--negative" : "ck-delta--zero" %>
|
|
70
|
-
<span class="ck-delta <%= dcls %>"><span class="ck-delta__arrow" aria-hidden="true"><%= d > 0 ? "▲" : d < 0 ? "▼" : "=" %></span><%= d > 0 ? "+#{d}" : d.to_s %></span>
|
|
74
|
+
<%= ck_delta_chip(pm[:delta]) %>
|
|
71
75
|
<% else %>
|
|
72
76
|
<span class="ck-cmp-score__none">—</span>
|
|
73
77
|
<% end %>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
<div class="ck-select" data-ck-select>
|
|
3
3
|
<input type="hidden" name="<%= name %>" value="<%= chosen && chosen.first %>" data-ck-select-value>
|
|
4
4
|
<button type="button" class="ck-select__trigger" data-ck-select-trigger
|
|
5
|
-
aria-haspopup="listbox" aria-expanded="false" aria-labelledby="<%= labelledby %>"
|
|
5
|
+
aria-haspopup="listbox" aria-expanded="false" aria-labelledby="<%= labelledby %>" <%= tag.attributes("aria-invalid": ("true" if local_assigns[:invalid])) %>>
|
|
6
6
|
<span class="ck-select__label" data-ck-select-label><%= chosen && chosen.last %></span>
|
|
7
7
|
<span class="ck-select__chevron" aria-hidden="true"><%= heroicon_tag "chevron-down" %></span>
|
|
8
8
|
</button>
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<!DOCTYPE html>
|
|
2
2
|
<html lang="en">
|
|
3
3
|
<head>
|
|
4
|
-
<title><%= [content_for(:ck_title).presence, "CompletionKit"].compact.join("
|
|
4
|
+
<title><%= [content_for(:ck_title).presence, "CompletionKit"].compact.join(" · ") %></title>
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
6
|
<%= csrf_meta_tags %>
|
|
7
7
|
<%= csp_meta_tag %>
|
data/config/locales/en.yml
CHANGED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class DropCompletionKitMcpSessions < ActiveRecord::Migration[8.1]
|
|
2
|
+
def up
|
|
3
|
+
drop_table :completion_kit_mcp_sessions, if_exists: true
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
def down
|
|
7
|
+
create_table :completion_kit_mcp_sessions do |t|
|
|
8
|
+
t.string :session_id, null: false
|
|
9
|
+
t.datetime :expires_at, null: false
|
|
10
|
+
t.timestamps
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
add_index :completion_kit_mcp_sessions, :session_id, unique: true
|
|
14
|
+
add_index :completion_kit_mcp_sessions, :expires_at
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -9,9 +9,9 @@ module CompletionKit
|
|
|
9
9
|
class UnknownModelError < ConfigurationError
|
|
10
10
|
attr_reader :model_name
|
|
11
11
|
|
|
12
|
-
def initialize(model_name = nil)
|
|
12
|
+
def initialize(model_name = nil, message = nil)
|
|
13
13
|
@model_name = model_name
|
|
14
|
-
super(%(No configured provider serves model "#{model_name}"))
|
|
14
|
+
super(message || %(No configured provider serves model "#{model_name}"))
|
|
15
15
|
end
|
|
16
16
|
end
|
|
17
17
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: completion-kit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.29.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Damien Bastin
|
|
@@ -264,13 +264,20 @@ files:
|
|
|
264
264
|
- app/services/completion_kit/checks/contains.rb
|
|
265
265
|
- app/services/completion_kit/checks/equals.rb
|
|
266
266
|
- app/services/completion_kit/checks/expected_resolver.rb
|
|
267
|
+
- app/services/completion_kit/checks/flag.rb
|
|
267
268
|
- app/services/completion_kit/checks/json_path_equals.rb
|
|
269
|
+
- app/services/completion_kit/checks/labels.rb
|
|
268
270
|
- app/services/completion_kit/checks/length_bounds.rb
|
|
271
|
+
- app/services/completion_kit/checks/list_overlap.rb
|
|
269
272
|
- app/services/completion_kit/checks/not_contains.rb
|
|
273
|
+
- app/services/completion_kit/checks/numeric_bounds.rb
|
|
274
|
+
- app/services/completion_kit/checks/numeric_equals.rb
|
|
275
|
+
- app/services/completion_kit/checks/numeric_value.rb
|
|
270
276
|
- app/services/completion_kit/checks/regex.rb
|
|
271
277
|
- app/services/completion_kit/checks/registry.rb
|
|
272
278
|
- app/services/completion_kit/checks/result.rb
|
|
273
279
|
- app/services/completion_kit/checks/target_resolver.rb
|
|
280
|
+
- app/services/completion_kit/checks/textual.rb
|
|
274
281
|
- app/services/completion_kit/checks/valid_json.rb
|
|
275
282
|
- app/services/completion_kit/csv_processor.rb
|
|
276
283
|
- app/services/completion_kit/dashboard_stats.rb
|
|
@@ -452,6 +459,8 @@ files:
|
|
|
452
459
|
- db/migrate/20260730000002_create_completion_kit_prompt_serves.rb
|
|
453
460
|
- db/migrate/20260804000001_add_judge_temperature_ignored_to_runs.rb
|
|
454
461
|
- db/migrate/20260804000002_default_runs_to_no_temperature.rb
|
|
462
|
+
- db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb
|
|
463
|
+
- db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb
|
|
455
464
|
- lib/completion-kit.rb
|
|
456
465
|
- lib/completion_kit.rb
|
|
457
466
|
- lib/completion_kit/concurrency_check.rb
|