completion-kit 0.28.46 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -3
- data/app/assets/javascripts/completion_kit/application.js +435 -23
- data/app/assets/stylesheets/completion_kit/application.css +44 -14
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/mcp_controller.rb +4 -0
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/runs_controller.rb +4 -1
- data/app/helpers/completion_kit/application_helper.rb +99 -45
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/dataset.rb +53 -4
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +8 -0
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +20 -9
- data/app/services/completion_kit/api_config.rb +2 -1
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +14 -4
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/csv_processor.rb +14 -2
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/model_discovery_service.rb +2 -1
- data/app/services/completion_kit/ollama_client.rb +19 -2
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/provider_endpoint.rb +10 -0
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/api_reference/_authentication.html.erb +2 -2
- data/app/views/completion_kit/api_reference/_body.html.erb +7 -30
- data/app/views/completion_kit/api_reference/_example.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_resource_card.html.erb +1 -1
- data/app/views/completion_kit/datasets/_form.html.erb +0 -28
- data/app/views/completion_kit/datasets/index.html.erb +1 -1
- data/app/views/completion_kit/metric_groups/_form.html.erb +1 -20
- data/app/views/completion_kit/metric_groups/index.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/metrics/_starter_card.html.erb +1 -1
- data/app/views/completion_kit/metrics/index.html.erb +2 -2
- data/app/views/completion_kit/metrics/show.html.erb +4 -14
- data/app/views/completion_kit/prompts/_form.html.erb +1 -1
- data/app/views/completion_kit/prompts/index.html.erb +3 -3
- data/app/views/completion_kit/prompts/show.html.erb +8 -8
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +4 -174
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -5
- data/app/views/completion_kit/runs/_row.html.erb +13 -11
- data/app/views/completion_kit/runs/_run_config.html.erb +2 -2
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/show.html.erb +5 -19
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/completion_kit/tags/index.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +2 -2
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/lib/completion_kit/version.rb +1 -1
- data/lib/completion_kit.rb +14 -0
- data/lib/generators/completion_kit/templates/initializer.rb +18 -0
- metadata +9 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f314f2989362808e829e3f1bbbfc5718eca193aa4602ae4b4b3e72bc6c9251c1
|
|
4
|
+
data.tar.gz: b9322db963ff8bb00660e09309940f692cab0fff289c8069b3be68ecf81db429
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bff1f44883f8a652ffde54da844bd626db029e432e91f5f79648b926b34ada993599b0a04176bfc842bd1c7da786dc9a05af5744779ba6e3fdca677e213639d1
|
|
7
|
+
data.tar.gz: bff0d10f9a3f7ccc857d52b5038ac06e4ccbc6e15e73195ff18c95a2f0223c4d223bdd17749f36d6933d7bb5925e3dd9e522cf4e012b26ac76a9c9b78c40a0c6
|
data/README.md
CHANGED
|
@@ -71,10 +71,33 @@ The engine mounts at `/completion_kit`. Generate / judge flows enqueue Active Jo
|
|
|
71
71
|
|
|
72
72
|
```erb
|
|
73
73
|
<%= stylesheet_link_tag "completion_kit/application", media: "all" %>
|
|
74
|
-
<%= javascript_include_tag "completion_kit/application", defer: true %>
|
|
74
|
+
<%= javascript_include_tag "completion_kit/application", defer: true, nonce: true %>
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
Without the JavaScript include, in-page behaviours silently fail: live tag-breadcrumb updates, relative-time ticking, CSV row hover-expand, model-refresh progress, focus-first-error, and local-time formatting.
|
|
77
|
+
Without the JavaScript include, in-page behaviours silently fail: clickable table rows, dialogs, copy buttons, the API token reveal, the run form, live run status, live tag-breadcrumb updates, relative-time ticking, CSV row hover-expand, model-refresh progress, focus-first-error, and local-time formatting.
|
|
78
|
+
|
|
79
|
+
**Content Security Policy.** The engine's pages have no inline event handlers or inline scripts, and the engine layout puts the request nonce on its script tags, so the pages work under a nonce-based `script-src`. The views still use inline `style` attributes, so leave `style-src` out of the nonce directives and allow `'unsafe-inline'` there for now.
|
|
80
|
+
|
|
81
|
+
Use a nonce that lasts the whole session, not a fresh random one per request. Turbo Drive re-creates the scripts in each new page body with the nonce from the `csp-nonce` meta tag, but the browser keeps enforcing the policy header of the first page it loaded, so a per-request nonce blocks nonce scripts after the first Drive visit.
|
|
82
|
+
|
|
83
|
+
Keep a random nonce in the session, and skip it for API controllers so cookie-less API and MCP calls get no session cookie, as the example below does. The Rails default, `->(request) { request.session.id.to_s }`, also lasts the session, but it is empty on a brand-new visitor's first response, so a policy that admits scripts only by nonce blocks that first page's scripts until the session exists.
|
|
84
|
+
|
|
85
|
+
```ruby
|
|
86
|
+
Rails.application.configure do
|
|
87
|
+
config.content_security_policy do |policy|
|
|
88
|
+
policy.default_src :self
|
|
89
|
+
policy.script_src :self
|
|
90
|
+
policy.style_src :self, :unsafe_inline
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
config.content_security_policy_nonce_generator = lambda do |request|
|
|
94
|
+
next if request.controller_instance.is_a?(ActionController::API)
|
|
95
|
+
|
|
96
|
+
request.session[:csp_nonce] ||= SecureRandom.base64(16)
|
|
97
|
+
end
|
|
98
|
+
config.content_security_policy_nonce_directives = %w(script-src)
|
|
99
|
+
end
|
|
100
|
+
```
|
|
78
101
|
|
|
79
102
|
## Providers
|
|
80
103
|
|
|
@@ -84,12 +107,14 @@ CompletionKit discovers available models from each provider's API automatically.
|
|
|
84
107
|
|----------|----------|----------------|
|
|
85
108
|
| **OpenAI** | `OPENAI_API_KEY` | GPT-5, GPT-4.1, GPT-4o, etc. |
|
|
86
109
|
| **Anthropic** | `ANTHROPIC_API_KEY` | Claude Opus, Sonnet, Haiku |
|
|
87
|
-
| **Ollama / local endpoint** | `OLLAMA_API_ENDPOINT` (default: `http://localhost:11434/v1`) | Any model you've `ollama pull`-ed, or any OpenAI-compatible local server (vLLM, LM Studio, llama.cpp) |
|
|
110
|
+
| **Ollama / local endpoint** | `OLLAMA_API_ENDPOINT` (default: `http://localhost:11434/v1`), `OLLAMA_API_KEY` (optional) | Any model you've `ollama pull`-ed, or any OpenAI-compatible local server (vLLM, LM Studio, llama.cpp) |
|
|
88
111
|
| **OpenRouter** | `OPENROUTER_API_KEY` | 100+ models from 30+ providers through one API key |
|
|
89
112
|
| **Azure AI Foundry** | No env var — add via the Providers UI / a `ProviderCredential` (`api_key` + `api_endpoint` required, `api_version` optional) | Azure OpenAI deployments and Foundry projects |
|
|
90
113
|
|
|
91
114
|
Set these as environment variables or configure them in the generated initializer. You can also add provider credentials through the web UI under Settings.
|
|
92
115
|
|
|
116
|
+
The Ollama key from `OLLAMA_API_KEY` or `config.ollama_api_key` goes only to the Ollama endpoint from `OLLAMA_API_ENDPOINT` or `config.ollama_api_endpoint`, or to `http://localhost:11434/v1` when neither is set and `config.allow_loopback_endpoints` is on, which is the default. A provider credential with no key of its own gets that key only while it points at that same endpoint. Point it anywhere else and CompletionKit sends no key, so store the key on the credential itself. A key stored on a credential always goes to that credential's endpoint. Two endpoints match when their scheme, host, port and path match. The case of the scheme and host, a default port written out, a trailing slash and a trailing `/v1` make no difference. A blank stored key counts as no key.
|
|
117
|
+
|
|
93
118
|
### Encryption
|
|
94
119
|
|
|
95
120
|
Provider API keys are encrypted at rest using [Active Record encryption](https://guides.rubyonrails.org/active_record_encryption.html). You need three encryption keys configured before the app will boot in production.
|
|
@@ -169,6 +194,26 @@ end
|
|
|
169
194
|
|
|
170
195
|
Limiting uses `Rails.cache`. A shared cache store (Solid Cache, Redis) throttles accurately across multiple app instances; a per-process store still throttles each instance independently.
|
|
171
196
|
|
|
197
|
+
## Input limits
|
|
198
|
+
|
|
199
|
+
Uploads, prompt templates, and regex check patterns have limits, so one large or hostile input cannot tie up a worker. Tune them in the initializer:
|
|
200
|
+
|
|
201
|
+
```ruby
|
|
202
|
+
CompletionKit.configure do |c|
|
|
203
|
+
c.max_upload_bytes = 25 * 1024 * 1024 # largest dataset CSV or import file (default 25 MB)
|
|
204
|
+
c.max_template_bytes = 1024 * 1024 # largest prompt template (default 1 MB)
|
|
205
|
+
c.regex_timeout = 1.0 # seconds a regex check may spend matching one row (default 1.0)
|
|
206
|
+
end
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
A prompt whose template is over `max_template_bytes` fails validation when its template is saved. Prompts saved before a lower limit keep working until their template changes.
|
|
210
|
+
|
|
211
|
+
The first line of a dataset's CSV is its header row. A dataset fails validation when its CSV is saved with a header row that is blank or longer than 64 KB. The 64 KB limit is fixed, not a setting.
|
|
212
|
+
|
|
213
|
+
A dataset also fails validation when its CSV is saved with bytes that are not valid UTF-8, so export spreadsheets as UTF-8 CSV.
|
|
214
|
+
|
|
215
|
+
A regex check that runs past its time limit fails that row and says it timed out. The limit is `regex_timeout`, or your app's `Regexp.timeout` when that is shorter, so the engine never loosens a stricter limit your app set. The engine sets the limit on each pattern and never changes the process-wide `Regexp.timeout`. Set `regex_timeout` to a finite number of seconds of at least `0.001`, or to `nil` to use only `Regexp.timeout`. Any other value, such as `0` or `Float::INFINITY`, raises an `ArgumentError` when the initializer runs.
|
|
216
|
+
|
|
172
217
|
## How it works
|
|
173
218
|
|
|
174
219
|
<p align="center">
|
|
@@ -8,6 +8,8 @@ document.addEventListener("turbo:load", function() {
|
|
|
8
8
|
if (ckDiscoveringInDom()) ckStartDiscoveryPolling(0);
|
|
9
9
|
});
|
|
10
10
|
|
|
11
|
+
document.addEventListener("turbo:render", ckAutoFocusFirstError);
|
|
12
|
+
|
|
11
13
|
document.addEventListener("input", function(e) {
|
|
12
14
|
if (!e.target || e.target.id !== "tag_name") return;
|
|
13
15
|
var text = document.getElementById("tag-pill-text");
|
|
@@ -214,37 +216,56 @@ document.addEventListener("click", function(e) {
|
|
|
214
216
|
});
|
|
215
217
|
});
|
|
216
218
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
valid_json: [],
|
|
223
|
-
json_path_equals: ["json_path", "expected"],
|
|
224
|
-
length_bounds: ["min", "max"]
|
|
225
|
-
};
|
|
219
|
+
function ckCheckEditor(scope) {
|
|
220
|
+
if (!scope) return null;
|
|
221
|
+
if (scope.matches && scope.matches("[data-ck-check-form]")) return scope;
|
|
222
|
+
return scope.querySelector ? scope.querySelector("[data-ck-check-form]") : null;
|
|
223
|
+
}
|
|
226
224
|
|
|
227
|
-
|
|
225
|
+
function ckCheckForm(editor) {
|
|
226
|
+
try {
|
|
227
|
+
return JSON.parse(editor.getAttribute("data-ck-check-form"));
|
|
228
|
+
} catch (e) {
|
|
229
|
+
return null;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function ckApplyCheckLabels(editor, form, kind) {
|
|
234
|
+
var labels = form.labels[kind] || {};
|
|
235
|
+
var hints = form.hints[kind] || {};
|
|
236
|
+
editor.querySelectorAll("[data-ck-check-label]").forEach(function(node) {
|
|
237
|
+
var key = node.getAttribute("data-ck-check-label");
|
|
238
|
+
node.textContent = labels[key] || form.defaultLabels[key] || "";
|
|
239
|
+
});
|
|
240
|
+
editor.querySelectorAll("[data-ck-check-hint]").forEach(function(node) {
|
|
241
|
+
var key = node.getAttribute("data-ck-check-hint");
|
|
242
|
+
node.textContent = hints[key] || form.defaultHints[key] || "";
|
|
243
|
+
});
|
|
244
|
+
}
|
|
228
245
|
|
|
229
246
|
function ckApplyCheckFields(scope) {
|
|
230
|
-
|
|
231
|
-
|
|
247
|
+
var editor = ckCheckEditor(scope);
|
|
248
|
+
if (!editor) return;
|
|
249
|
+
var form = ckCheckForm(editor);
|
|
250
|
+
if (!form) return;
|
|
251
|
+
var kindSelect = editor.querySelector('[name="metric[check_config][check_kind]"]');
|
|
232
252
|
if (!kindSelect) return;
|
|
233
253
|
var kind = kindSelect.value;
|
|
234
|
-
var visible = (
|
|
235
|
-
var targetSelect =
|
|
254
|
+
var visible = (form.fields[kind] || []).slice();
|
|
255
|
+
var targetSelect = editor.querySelector('[name="metric[check_config][target]"]');
|
|
236
256
|
var targetIsJsonPath = !!(targetSelect && targetSelect.value === "json_path");
|
|
257
|
+
var expectedKey = form.expected[kind];
|
|
237
258
|
|
|
238
|
-
if (
|
|
259
|
+
if (expectedKey) {
|
|
239
260
|
visible.push("compare_to");
|
|
240
|
-
var comparison =
|
|
261
|
+
var comparison = editor.querySelector('[name="metric[check_config][compare_to]"]:checked');
|
|
241
262
|
if (comparison && comparison.value === "expected") {
|
|
242
|
-
visible = visible.filter(function(key) { return key !==
|
|
263
|
+
visible = visible.filter(function(key) { return key !== expectedKey; });
|
|
243
264
|
visible.push("expected_path");
|
|
244
265
|
}
|
|
245
266
|
}
|
|
246
267
|
|
|
247
|
-
|
|
268
|
+
editor.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
|
|
248
269
|
var key = field.getAttribute("data-ck-check-field");
|
|
249
270
|
var show;
|
|
250
271
|
if (key === "target_path") {
|
|
@@ -254,6 +275,8 @@ function ckApplyCheckFields(scope) {
|
|
|
254
275
|
}
|
|
255
276
|
field.hidden = !show;
|
|
256
277
|
});
|
|
278
|
+
|
|
279
|
+
ckApplyCheckLabels(editor, form, kind);
|
|
257
280
|
}
|
|
258
281
|
|
|
259
282
|
function ckApplyMetricType(group) {
|
|
@@ -271,14 +294,17 @@ function ckApplyMetricType(group) {
|
|
|
271
294
|
ckApplyCheckFields(scope);
|
|
272
295
|
}
|
|
273
296
|
|
|
274
|
-
|
|
297
|
+
function ckApplyMetricForms() {
|
|
275
298
|
document.querySelectorAll("[data-ck-metric-type]").forEach(function(group) {
|
|
276
299
|
ckApplyMetricType(group);
|
|
277
300
|
});
|
|
278
|
-
document.querySelectorAll(
|
|
301
|
+
document.querySelectorAll("[data-ck-check-form]").forEach(function(editor) {
|
|
279
302
|
ckApplyCheckFields(editor);
|
|
280
303
|
});
|
|
281
|
-
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
document.addEventListener("turbo:load", ckApplyMetricForms);
|
|
307
|
+
document.addEventListener("turbo:render", ckApplyMetricForms);
|
|
282
308
|
|
|
283
309
|
document.addEventListener("change", function(e) {
|
|
284
310
|
var target = e.target;
|
|
@@ -305,11 +331,14 @@ function ckApplyProviderFields(scope) {
|
|
|
305
331
|
});
|
|
306
332
|
}
|
|
307
333
|
|
|
308
|
-
|
|
334
|
+
function ckApplyProviderForms() {
|
|
309
335
|
document.querySelectorAll("[data-ck-provider-select]").forEach(function(select) {
|
|
310
336
|
ckApplyProviderFields(select.closest("form") || document);
|
|
311
337
|
});
|
|
312
|
-
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
document.addEventListener("turbo:load", ckApplyProviderForms);
|
|
341
|
+
document.addEventListener("turbo:render", ckApplyProviderForms);
|
|
313
342
|
|
|
314
343
|
document.addEventListener("change", function(e) {
|
|
315
344
|
var target = e.target;
|
|
@@ -454,3 +483,386 @@ function ckInstallConfirm() {
|
|
|
454
483
|
}
|
|
455
484
|
document.addEventListener("turbo:load", ckInstallConfirm);
|
|
456
485
|
ckInstallConfirm();
|
|
486
|
+
|
|
487
|
+
var ckRowIgnoreSelector = "a, button, input, select, textarea, label, summary, form, dialog, [data-ck-row-ignore]";
|
|
488
|
+
|
|
489
|
+
function ckRowHref(e) {
|
|
490
|
+
var target = e.target;
|
|
491
|
+
if (!target || !target.closest) return null;
|
|
492
|
+
var row = target.closest("[data-ck-href]");
|
|
493
|
+
if (!row) return null;
|
|
494
|
+
var inner = target.closest(ckRowIgnoreSelector);
|
|
495
|
+
if (inner && row.contains(inner)) return null;
|
|
496
|
+
return row.getAttribute("data-ck-href");
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
document.addEventListener("click", function(e) {
|
|
500
|
+
var href = ckRowHref(e);
|
|
501
|
+
if (!href) return;
|
|
502
|
+
if (e.metaKey || e.ctrlKey || e.shiftKey) {
|
|
503
|
+
window.open(href, "_blank");
|
|
504
|
+
} else if (window.Turbo && typeof window.Turbo.visit === "function") {
|
|
505
|
+
window.Turbo.visit(href);
|
|
506
|
+
} else {
|
|
507
|
+
window.location.assign(href);
|
|
508
|
+
}
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
document.addEventListener("auxclick", function(e) {
|
|
512
|
+
if (e.button !== 1) return;
|
|
513
|
+
var href = ckRowHref(e);
|
|
514
|
+
if (href) window.open(href, "_blank");
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
document.addEventListener("click", function(e) {
|
|
518
|
+
if (e.target && e.target.closest && e.target.closest("[data-ck-stop-propagation]")) e.stopPropagation();
|
|
519
|
+
});
|
|
520
|
+
|
|
521
|
+
function ckShowDialog(dialog) {
|
|
522
|
+
if (dialog && typeof dialog.showModal === "function" && !dialog.open) dialog.showModal();
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
document.addEventListener("click", function(e) {
|
|
526
|
+
var target = e.target;
|
|
527
|
+
if (!target || !target.closest) return;
|
|
528
|
+
var opener = target.closest("[data-ck-dialog-open]");
|
|
529
|
+
if (opener) {
|
|
530
|
+
ckShowDialog(document.getElementById(opener.getAttribute("data-ck-dialog-open")));
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
var closer = target.closest("[data-ck-dialog-close]");
|
|
534
|
+
if (closer) {
|
|
535
|
+
var dialog = closer.closest("dialog");
|
|
536
|
+
if (dialog) dialog.close();
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
if (target.matches("dialog[data-ck-dialog]")) target.close();
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
document.addEventListener("turbo:load", function() {
|
|
543
|
+
ckShowDialog(document.querySelector("dialog[data-ck-dialog-autoopen]"));
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
function ckFlashCopied(el) {
|
|
547
|
+
el.classList.add("ck-api-copy--done");
|
|
548
|
+
setTimeout(function() { el.classList.remove("ck-api-copy--done"); }, 1500);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function ckCopyText(el, text) {
|
|
552
|
+
navigator.clipboard.writeText(text).then(function() {
|
|
553
|
+
if (el.hasAttribute("data-ck-copy-feedback")) ckFlashCopied(el);
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
function ckToggleToken(el) {
|
|
558
|
+
var showing = el.textContent === el.dataset.masked;
|
|
559
|
+
el.textContent = showing ? el.dataset.real : el.dataset.masked;
|
|
560
|
+
el.setAttribute("aria-pressed", showing ? "true" : "false");
|
|
561
|
+
el.setAttribute("aria-label", showing ? "Hide API token" : "Reveal API token");
|
|
562
|
+
navigator.clipboard.writeText(el.dataset.real).then(function() { ckFlashCopied(el); });
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function ckCopyExample(btn) {
|
|
566
|
+
var text = btn.closest(".ck-api-example").querySelector("pre").textContent;
|
|
567
|
+
var realToken = btn.dataset.realToken;
|
|
568
|
+
var displayToken = btn.dataset.displayToken;
|
|
569
|
+
if (realToken && displayToken) text = text.split(displayToken).join(realToken);
|
|
570
|
+
navigator.clipboard.writeText(text).then(function() { ckFlashCopied(btn); });
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
document.addEventListener("click", function(e) {
|
|
574
|
+
var target = e.target;
|
|
575
|
+
if (!target || !target.closest) return;
|
|
576
|
+
var toggle = target.closest("[data-ck-token-toggle]");
|
|
577
|
+
if (toggle) { ckToggleToken(toggle); return; }
|
|
578
|
+
var example = target.closest("[data-ck-copy-example]");
|
|
579
|
+
if (example) { ckCopyExample(example); return; }
|
|
580
|
+
var literal = target.closest("[data-ck-copy-text]");
|
|
581
|
+
if (literal) { ckCopyText(literal, literal.getAttribute("data-ck-copy-text")); return; }
|
|
582
|
+
var copier = target.closest("[data-ck-copy]");
|
|
583
|
+
if (!copier) return;
|
|
584
|
+
var source = document.getElementById(copier.getAttribute("data-ck-copy"));
|
|
585
|
+
if (source) ckCopyText(copier, source.textContent);
|
|
586
|
+
});
|
|
587
|
+
|
|
588
|
+
document.addEventListener("click", function(e) {
|
|
589
|
+
var target = e.target;
|
|
590
|
+
if (!target || !target.closest) return;
|
|
591
|
+
if (target.closest("[data-ck-refresh-models]")) {
|
|
592
|
+
ckRefreshModels();
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
var refresher = target.closest("[data-ck-provider-refresh]");
|
|
596
|
+
if (!refresher) return;
|
|
597
|
+
e.preventDefault();
|
|
598
|
+
fetch(refresher.getAttribute("data-ck-provider-refresh"), {
|
|
599
|
+
method: "POST",
|
|
600
|
+
headers: { "X-CSRF-Token": document.querySelector('meta[name="csrf-token"]').getAttribute("content") }
|
|
601
|
+
});
|
|
602
|
+
ckStartDiscoveryPolling(8000);
|
|
603
|
+
});
|
|
604
|
+
|
|
605
|
+
document.addEventListener("click", function(e) {
|
|
606
|
+
var toggle = e.target && e.target.closest && e.target.closest("[data-ck-prompt-toggle]");
|
|
607
|
+
if (!toggle) return;
|
|
608
|
+
var text = document.getElementById(toggle.getAttribute("aria-controls"));
|
|
609
|
+
if (!text) return;
|
|
610
|
+
var expanded = text.classList.toggle("ck-prompt-preview__text--expanded");
|
|
611
|
+
toggle.firstChild.textContent = expanded ? "Show less" : "Show more";
|
|
612
|
+
toggle.setAttribute("aria-expanded", expanded ? "true" : "false");
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
function ckToggleMetricGroupFilter(btn) {
|
|
616
|
+
btn.classList.toggle("tag-mark--off");
|
|
617
|
+
var bar = btn.closest("[data-metric-tag-filter]");
|
|
618
|
+
var grid = bar && bar.parentElement.querySelector("[data-metric-checkboxes]");
|
|
619
|
+
if (!grid) return;
|
|
620
|
+
var active = Array.prototype.map.call(
|
|
621
|
+
bar.querySelectorAll(".ck-metric-tag-filter__chip:not(.tag-mark--off)"),
|
|
622
|
+
function(b) { return b.getAttribute("data-filter-tag"); }
|
|
623
|
+
);
|
|
624
|
+
grid.querySelectorAll(".ck-checkbox-label").forEach(function(lbl) {
|
|
625
|
+
var tags = (lbl.getAttribute("data-metric-tags") || "").split(",").filter(Boolean);
|
|
626
|
+
var visible = active.length === 0 || tags.some(function(t) { return active.indexOf(t) !== -1; });
|
|
627
|
+
lbl.style.display = visible ? "" : "none";
|
|
628
|
+
});
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
document.addEventListener("click", function(e) {
|
|
632
|
+
var target = e.target;
|
|
633
|
+
if (!target || !target.closest) return;
|
|
634
|
+
var chip = target.closest("[data-metric-tag-filter] [data-filter-tag]");
|
|
635
|
+
if (chip) {
|
|
636
|
+
ckToggleMetricGroupFilter(chip);
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
var group = target.closest("[data-metric-group]");
|
|
640
|
+
if (group) ckToggleMetricGroup(group);
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
function ckUpdateRunForm() {
|
|
644
|
+
var promptEl = document.getElementById('run_prompt_id');
|
|
645
|
+
var judgeEl = document.getElementById('run_judge_model');
|
|
646
|
+
var judgeOnlyEl = document.getElementById('run_judge_only');
|
|
647
|
+
var judgeOnly = !!(judgeOnlyEl && judgeOnlyEl.checked);
|
|
648
|
+
var promptField = document.getElementById('prompt-field');
|
|
649
|
+
var outputColumnField = document.getElementById('output-column-field');
|
|
650
|
+
var outputColumnEl = document.getElementById('run_output_column');
|
|
651
|
+
if (promptField) promptField.hidden = judgeOnly;
|
|
652
|
+
if (outputColumnField) outputColumnField.hidden = !judgeOnly;
|
|
653
|
+
if (judgeOnly && promptEl) promptEl.value = '';
|
|
654
|
+
|
|
655
|
+
var prompt = promptEl ? promptEl.value : '';
|
|
656
|
+
var judge = judgeEl ? judgeEl.value : '';
|
|
657
|
+
var judgeListed = !!judgeEl && judgeEl.type !== 'hidden';
|
|
658
|
+
var metrics = document.querySelectorAll('input[name="run[metric_ids][]"]:checked');
|
|
659
|
+
var judgeField = document.getElementById('judge-field');
|
|
660
|
+
var metricsField = document.getElementById('metrics-field');
|
|
661
|
+
var judgeHint = document.getElementById('judge-hint');
|
|
662
|
+
var metricsHint = document.getElementById('metrics-hint');
|
|
663
|
+
var submitBtn = document.getElementById('run-submit');
|
|
664
|
+
|
|
665
|
+
if (judgeField) judgeField.className = 'ck-field';
|
|
666
|
+
if (metricsField) metricsField.className = 'ck-field';
|
|
667
|
+
if (judgeHint) judgeHint.textContent = '';
|
|
668
|
+
if (metricsHint) metricsHint.textContent = '';
|
|
669
|
+
|
|
670
|
+
var judgedMetrics = Array.prototype.filter.call(metrics, function(cb) {
|
|
671
|
+
var lbl = cb.closest('[data-needs-judge]');
|
|
672
|
+
return !!lbl && lbl.dataset.needsJudge === '1';
|
|
673
|
+
});
|
|
674
|
+
var judgeUnpaired = judgeListed && !!judge && metrics.length === 0;
|
|
675
|
+
var metricsUnpaired = judgeListed && !judge && judgedMetrics.length > 0;
|
|
676
|
+
if (judgeListed && judge && metrics.length > 0 && judgedMetrics.length === 0) {
|
|
677
|
+
if (judgeHint) judgeHint.textContent = 'Checks need no judge, so this run will not use it.';
|
|
678
|
+
} else if (judgeListed && judge && metrics.length > 0) {
|
|
679
|
+
if (judgeHint) judgeHint.textContent = judgeOnly ? 'Existing outputs will be judged automatically.' : 'Responses will be generated then judged automatically.';
|
|
680
|
+
} else if (judgeUnpaired) {
|
|
681
|
+
if (judgeField) judgeField.className = 'ck-field ck-field--info';
|
|
682
|
+
if (metricsField) metricsField.className = 'ck-field ck-field--info';
|
|
683
|
+
if (metricsHint) metricsHint.textContent = 'Select at least one metric or group to enable judging.';
|
|
684
|
+
} else if (metricsUnpaired) {
|
|
685
|
+
if (judgeField) judgeField.className = 'ck-field ck-field--info';
|
|
686
|
+
if (judgeHint) judgeHint.textContent = 'Select a judge model to enable judging.';
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
var datasetEl = document.getElementById('run_dataset_id');
|
|
690
|
+
var datasetHint = document.getElementById('dataset-hint');
|
|
691
|
+
var datasetField = document.getElementById('dataset-field');
|
|
692
|
+
var dataset = datasetEl ? datasetEl.value : '';
|
|
693
|
+
var selectedOption = promptEl ? promptEl.options[promptEl.selectedIndex] : null;
|
|
694
|
+
var hasVars = selectedOption && selectedOption.dataset.hasVariables === '1';
|
|
695
|
+
var promptVars = (selectedOption && selectedOption.dataset.variables ? selectedOption.dataset.variables.split(/,\s*/) : []).filter(Boolean);
|
|
696
|
+
|
|
697
|
+
var missingVars = [];
|
|
698
|
+
if (hasVars && dataset && datasetEl) {
|
|
699
|
+
var datasetOption = datasetEl.options[datasetEl.selectedIndex];
|
|
700
|
+
var headers = (datasetOption && datasetOption.dataset.headers ? datasetOption.dataset.headers.split(/,\s*/) : []).filter(Boolean);
|
|
701
|
+
missingVars = promptVars.filter(function(v) { return headers.indexOf(v) === -1; });
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
if (datasetField) datasetField.className = 'ck-field';
|
|
705
|
+
if (datasetHint) datasetHint.textContent = '';
|
|
706
|
+
if (missingVars.length > 0) {
|
|
707
|
+
if (datasetField) datasetField.className = 'ck-field ck-field--error';
|
|
708
|
+
if (datasetHint) datasetHint.textContent = 'Dataset is missing ' + (missingVars.length === 1 ? 'column' : 'columns') + ' the prompt needs: ' + missingVars.join(', ');
|
|
709
|
+
} else if (hasVars && !dataset) {
|
|
710
|
+
if (datasetField) datasetField.className = 'ck-field ck-field--info';
|
|
711
|
+
if (datasetHint) datasetHint.textContent = 'This prompt uses variables. Select a dataset to provide values.';
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
var expectedWarn = document.getElementById('expected-column-warn');
|
|
715
|
+
var expectedField = document.getElementById('expected-column-field');
|
|
716
|
+
var expectedColEl = document.getElementById('run_expected_column');
|
|
717
|
+
if (expectedField) expectedField.className = 'ck-field';
|
|
718
|
+
if (expectedWarn) expectedWarn.textContent = '';
|
|
719
|
+
var wantsExpected = false;
|
|
720
|
+
document.querySelectorAll('input[name="run[metric_ids][]"]:checked').forEach(function(cb) {
|
|
721
|
+
var lbl = cb.closest('[data-compare-expected]');
|
|
722
|
+
if (lbl && lbl.dataset.compareExpected === '1') wantsExpected = true;
|
|
723
|
+
});
|
|
724
|
+
if (wantsExpected && dataset && datasetEl) {
|
|
725
|
+
var expOption = datasetEl.options[datasetEl.selectedIndex];
|
|
726
|
+
var expHeaders = (expOption && expOption.dataset.headers ? expOption.dataset.headers.split(/,\s*/) : []).filter(Boolean);
|
|
727
|
+
var expCol = ((expectedColEl && expectedColEl.value) || 'expected_output').trim();
|
|
728
|
+
if (expHeaders.indexOf(expCol) === -1) {
|
|
729
|
+
if (expectedField) expectedField.className = 'ck-field ck-field--error';
|
|
730
|
+
if (expectedWarn) expectedWarn.textContent = "A selected check grades against each row's expected value, but this dataset has no \"" + expCol + "\" column, so every row would fail. Add the column or point this at the right one.";
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
var summary = document.getElementById('prompt-summary');
|
|
735
|
+
if (summary) {
|
|
736
|
+
if (selectedOption && selectedOption.value) {
|
|
737
|
+
var desc = selectedOption.dataset.description || '';
|
|
738
|
+
var tmpl = selectedOption.dataset.templatePreview || '';
|
|
739
|
+
var descEl = document.getElementById('prompt-summary-description');
|
|
740
|
+
descEl.textContent = desc;
|
|
741
|
+
descEl.hidden = !desc;
|
|
742
|
+
document.getElementById('prompt-summary-template').textContent = tmpl;
|
|
743
|
+
summary.hidden = false;
|
|
744
|
+
} else {
|
|
745
|
+
summary.hidden = true;
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
var valid;
|
|
750
|
+
if (judgeOnly) {
|
|
751
|
+
valid = !!dataset;
|
|
752
|
+
if (dataset && datasetEl && outputColumnEl) {
|
|
753
|
+
var headersJudge = (datasetEl.options[datasetEl.selectedIndex] && datasetEl.options[datasetEl.selectedIndex].dataset.headers ? datasetEl.options[datasetEl.selectedIndex].dataset.headers.split(/,\s*/) : []).filter(Boolean);
|
|
754
|
+
var col = (outputColumnEl.value || 'actual_output').trim();
|
|
755
|
+
if (col === '' || headersJudge.indexOf(col) === -1) {
|
|
756
|
+
valid = false;
|
|
757
|
+
if (datasetField) datasetField.className = 'ck-field ck-field--error';
|
|
758
|
+
if (datasetHint) datasetHint.textContent = "Dataset has no \"" + col + "\" column. Pick a different output column or dataset.";
|
|
759
|
+
}
|
|
760
|
+
} else if (!dataset) {
|
|
761
|
+
if (datasetField) datasetField.className = 'ck-field ck-field--info';
|
|
762
|
+
if (datasetHint) datasetHint.textContent = 'Skip generation and score responses you already have from a dataset column. Works with rubric metrics or deterministic checks.';
|
|
763
|
+
}
|
|
764
|
+
} else {
|
|
765
|
+
valid = prompt !== '';
|
|
766
|
+
if (hasVars && !dataset) valid = false;
|
|
767
|
+
if (missingVars.length > 0) valid = false;
|
|
768
|
+
}
|
|
769
|
+
if (judgeUnpaired || metricsUnpaired) valid = false;
|
|
770
|
+
if (submitBtn) submitBtn.disabled = !valid;
|
|
771
|
+
|
|
772
|
+
ckUpdateMetricGroupsState();
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
function ckToggleMetricGroup(button) {
|
|
776
|
+
var ids = (button.getAttribute('data-metric-ids') || '').split(',').filter(Boolean);
|
|
777
|
+
var allChecked = ids.every(function(id) {
|
|
778
|
+
var cb = document.getElementById('run_metric_' + id);
|
|
779
|
+
return cb && cb.checked;
|
|
780
|
+
});
|
|
781
|
+
ids.forEach(function(id) {
|
|
782
|
+
var cb = document.getElementById('run_metric_' + id);
|
|
783
|
+
if (cb) cb.checked = !allChecked;
|
|
784
|
+
});
|
|
785
|
+
ckUpdateRunForm();
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
function ckUpdateMetricGroupsState() {
|
|
789
|
+
document.querySelectorAll('[data-metric-group]').forEach(function(btn) {
|
|
790
|
+
var ids = (btn.getAttribute('data-metric-ids') || '').split(',').filter(Boolean);
|
|
791
|
+
if (ids.length === 0) return;
|
|
792
|
+
var allChecked = ids.every(function(id) {
|
|
793
|
+
var cb = document.getElementById('run_metric_' + id);
|
|
794
|
+
return cb && cb.checked;
|
|
795
|
+
});
|
|
796
|
+
btn.classList.toggle('ck-metric-group-pill--active', allChecked);
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
var ckRunFormChangeIds = ["run_judge_model", "run_prompt_id", "run_dataset_id", "run_judge_only"];
|
|
801
|
+
|
|
802
|
+
document.addEventListener("change", function(e) {
|
|
803
|
+
var target = e.target;
|
|
804
|
+
if (!target) return;
|
|
805
|
+
if (ckRunFormChangeIds.indexOf(target.id) !== -1 || (target.type === "checkbox" && target.name === "run[metric_ids][]")) ckUpdateRunForm();
|
|
806
|
+
});
|
|
807
|
+
|
|
808
|
+
document.addEventListener("input", function(e) {
|
|
809
|
+
if (e.target && e.target.id === "run_output_column") ckUpdateRunForm();
|
|
810
|
+
});
|
|
811
|
+
|
|
812
|
+
function ckInitRunForm() {
|
|
813
|
+
if (document.getElementById("run_judge_only")) ckUpdateRunForm();
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
function ckDatasetHeaderPreview() {
|
|
817
|
+
var area = document.getElementById("dataset_csv_data");
|
|
818
|
+
var box = document.getElementById("dataset-header-preview");
|
|
819
|
+
var chips = document.getElementById("dataset-header-chips");
|
|
820
|
+
if (!area || !box || !chips) return;
|
|
821
|
+
var recognized = { expected_output: "answer key", actual_output: "pre-made output" };
|
|
822
|
+
var firstLine = (area.value.split(/\r?\n/).find(function(l) { return l.trim() !== ""; }) || "");
|
|
823
|
+
var headers = firstLine.split(",").map(function(h) { return h.trim().replace(/^"|"$/g, ""); }).filter(Boolean);
|
|
824
|
+
if (headers.length === 0) { box.hidden = true; chips.textContent = ""; return; }
|
|
825
|
+
box.hidden = false;
|
|
826
|
+
chips.textContent = "";
|
|
827
|
+
headers.forEach(function(h) {
|
|
828
|
+
var role = recognized[h];
|
|
829
|
+
var chip = document.createElement("span");
|
|
830
|
+
chip.className = "ck-header-chip" + (role ? " ck-header-chip--special" : "");
|
|
831
|
+
chip.textContent = role ? h + " · " + role : h;
|
|
832
|
+
chips.appendChild(chip);
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
document.addEventListener("input", function(e) {
|
|
837
|
+
if (e.target && e.target.id === "dataset_csv_data") ckDatasetHeaderPreview();
|
|
838
|
+
});
|
|
839
|
+
|
|
840
|
+
function ckInitPageForms() {
|
|
841
|
+
ckInitRunForm();
|
|
842
|
+
ckDatasetHeaderPreview();
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
document.addEventListener("turbo:load", ckInitPageForms);
|
|
846
|
+
document.addEventListener("turbo:render", ckInitPageForms);
|
|
847
|
+
|
|
848
|
+
function ckStartRunStatusPoll() {
|
|
849
|
+
clearTimeout(window.ckRunStatusKick);
|
|
850
|
+
clearInterval(window.ckRunStatusPoll);
|
|
851
|
+
window.ckRunStatusKick = null;
|
|
852
|
+
window.ckRunStatusPoll = null;
|
|
853
|
+
var marker = document.querySelector("[data-ck-run-status-url]");
|
|
854
|
+
if (!marker) return;
|
|
855
|
+
var url = marker.getAttribute("data-ck-run-status-url");
|
|
856
|
+
var refresh = function() {
|
|
857
|
+
fetch(url, { headers: { Accept: "text/vnd.turbo-stream.html" }, credentials: "same-origin" })
|
|
858
|
+
.then(function(r) { return r.ok ? r.text() : null; })
|
|
859
|
+
.then(function(html) { if (html) Turbo.renderStreamMessage(html); })
|
|
860
|
+
.catch(function() {});
|
|
861
|
+
};
|
|
862
|
+
window.ckRunStatusKick = setTimeout(refresh, 1000);
|
|
863
|
+
window.ckRunStatusPoll = setInterval(refresh, 15000);
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
document.addEventListener("turbo:load", ckStartRunStatusPoll);
|
|
867
|
+
|
|
868
|
+
ckInitPageForms();
|