completion-kit 0.20.3 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71e8645d7a790e2afc35e4fb7c8ca4ec5a6d08f51f3f84a2c289a2b6c76a9051
-  data.tar.gz: f5143766d938735ae5f2072516bc520c7d826894c02dc6a584bf62e40dd4b583
+  metadata.gz: 5db8eacad4c6487a87d970c4b942657ad5033b0647a846bdfba2a7c9548a54b0
+  data.tar.gz: 956456ed6c96d122283c6a437a91c062323153bee1e53ec32d10bae56bf2636c
 SHA512:
-  metadata.gz: 0a6c71340315065125e6bc520f80453132de07d858acd4e5327d2cc35d3b3420c920a2f2ccf148c696c20afdad15371f6323d27d85dd7f3d9baa8dd03d3f5386
-  data.tar.gz: 65c2978028e0897cb00e350f3a9e6af98f11e8d6fc6327da245acdbcc6de41bd783b8bbfe63b1cb76fc54e65debbd4488d71103f35d4222478c9611220d84a95
+  metadata.gz: e8b1f1e18e33a4875944bd779e107280b83acfc231cc4940b4f6fdcdd30d17d093c046fc0fb5836c2675c728681277006ccc54d9d0d72886ac09117d7c924c0d
+  data.tar.gz: c82ff9429c534d82ec3ca7b0acdcfa8ec1d50dfea091b4b04167a3dc65289b06bd3f4a5e2168654cbfb2a127fdd2fac69e77168847d899edd22b136e382e2560

data/app/assets/javascripts/completion_kit/application.js

@@ -210,6 +210,76 @@ document.addEventListener("click", function(e) {
   });
 });
 
+var CK_CHECK_FIELDS = {
+  contains: ["value", "case_sensitive", "trim"],
+  not_contains: ["value", "case_sensitive", "trim"],
+  equals: ["value", "case_sensitive", "trim"],
+  regex: ["pattern", "case_sensitive", "multiline"],
+  valid_json: [],
+  json_path_equals: ["json_path", "expected"],
+  length_bounds: ["min", "max"],
+  no_refusal: []
+};
+
+function ckApplyCheckFields(scope) {
+  if (!scope) return;
+  var kindSelect = scope.querySelector('[name="metric[check_config][check_kind]"]');
+  if (!kindSelect) return;
+  var visible = CK_CHECK_FIELDS[kindSelect.value];
+  var targetSelect = scope.querySelector('[name="metric[check_config][target]"]');
+  var targetIsJsonPath = !!(targetSelect && targetSelect.value === "json_path");
+  scope.querySelectorAll("[data-ck-check-field]").forEach(function(field) {
+    var key = field.getAttribute("data-ck-check-field");
+    var show;
+    if (key === "target_path") {
+      show = targetIsJsonPath;
+    } else if (!visible) {
+      show = true;
+    } else {
+      show = visible.indexOf(key) !== -1;
+    }
+    field.hidden = !show;
+  });
+}
+
+function ckApplyMetricType(group) {
+  var checked = group.querySelector('input[type="radio"]:checked');
+  if (!checked) return;
+  var value = checked.value;
+  var scope = group.closest("form") || document;
+  scope.querySelectorAll("[data-ck-metric-editor]").forEach(function(editor) {
+    var active = editor.getAttribute("data-ck-metric-editor") === value;
+    editor.hidden = !active;
+    editor.querySelectorAll("input, select, textarea").forEach(function(field) {
+      field.disabled = !active;
+    });
+  });
+  ckApplyCheckFields(scope);
+}
+
+document.addEventListener("turbo:load", function() {
+  document.querySelectorAll("[data-ck-metric-type]").forEach(function(group) {
+    ckApplyMetricType(group);
+  });
+  document.querySelectorAll('[data-ck-metric-editor="check"]').forEach(function(editor) {
+    ckApplyCheckFields(editor);
+  });
+});
+
+document.addEventListener("change", function(e) {
+  var target = e.target;
+  if (!target || !target.closest) return;
+  var group = target.closest("[data-ck-metric-type]");
+  if (group && target.type === "radio") {
+    ckApplyMetricType(group);
+    return;
+  }
+  if (target.name === "metric[check_config][check_kind]" || target.name === "metric[check_config][target]") {
+    var scope = target.closest('[data-ck-metric-editor="check"]') || target.closest("form");
+    ckApplyCheckFields(scope);
+  }
+});
+
 document.addEventListener("click", function(e) {
   var btn = e.target.closest("[data-ck-apply]");
   if (!btn) return;
@@ -226,3 +296,75 @@ document.addEventListener("click", function(e) {
   btn.textContent = "Applied ✓";
   field.focus({ preventScroll: true });
 });
+
+function ckSelectClose(sel) {
+  if (!sel.classList.contains("is-open")) return;
+  sel.classList.remove("is-open");
+  var menu = sel.querySelector("[data-ck-select-menu]");
+  var trigger = sel.querySelector("[data-ck-select-trigger]");
+  if (menu) menu.hidden = true;
+  if (trigger) trigger.setAttribute("aria-expanded", "false");
+}
+
+function ckSelectOpen(sel) {
+  document.querySelectorAll("[data-ck-select].is-open").forEach(ckSelectClose);
+  sel.classList.add("is-open");
+  var menu = sel.querySelector("[data-ck-select-menu]");
+  var trigger = sel.querySelector("[data-ck-select-trigger]");
+  if (menu) menu.hidden = false;
+  if (trigger) trigger.setAttribute("aria-expanded", "true");
+  var current = menu.querySelector('[aria-selected="true"]') || menu.querySelector('[role="option"]');
+  if (current) current.focus();
+}
+
+function ckSelectChoose(sel, option) {
+  var input = sel.querySelector("[data-ck-select-value]");
+  var label = sel.querySelector("[data-ck-select-label]");
+  var text = option.querySelector("[data-ck-select-text]");
+  sel.querySelectorAll('[role="option"]').forEach(function(o) {
+    o.setAttribute("aria-selected", o === option ? "true" : "false");
+  });
+  input.value = option.getAttribute("data-value");
+  if (text) label.textContent = text.textContent;
+  input.dispatchEvent(new Event("change", { bubbles: true }));
+  ckSelectClose(sel);
+  var trigger = sel.querySelector("[data-ck-select-trigger]");
+  if (trigger) trigger.focus();
+}
+
+document.addEventListener("click", function(e) {
+  var trigger = e.target.closest("[data-ck-select-trigger]");
+  if (trigger) {
+    var sel = trigger.closest("[data-ck-select]");
+    if (sel.classList.contains("is-open")) { ckSelectClose(sel); } else { ckSelectOpen(sel); }
+    return;
+  }
+  var option = e.target.closest('[data-ck-select] [role="option"]');
+  if (option) {
+    ckSelectChoose(option.closest("[data-ck-select]"), option);
+    return;
+  }
+  document.querySelectorAll("[data-ck-select].is-open").forEach(function(sel) {
+    if (!sel.contains(e.target)) ckSelectClose(sel);
+  });
+});
+
+document.addEventListener("keydown", function(e) {
+  var sel = e.target.closest("[data-ck-select]");
+  if (!sel) return;
+  var trigger = sel.querySelector("[data-ck-select-trigger]");
+  var options = Array.prototype.slice.call(sel.querySelectorAll('[role="option"]'));
+  var open = sel.classList.contains("is-open");
+  if (e.target === trigger && !open) {
+    if (e.key === "ArrowDown" || e.key === "Enter" || e.key === " ") { e.preventDefault(); ckSelectOpen(sel); }
+    return;
+  }
+  if (!open) return;
+  var idx = options.indexOf(document.activeElement);
+  if (e.key === "ArrowDown") { e.preventDefault(); (options[idx + 1] || options[0]).focus(); }
+  else if (e.key === "ArrowUp") { e.preventDefault(); (options[idx - 1] || options[options.length - 1]).focus(); }
+  else if (e.key === "Home") { e.preventDefault(); options[0].focus(); }
+  else if (e.key === "End") { e.preventDefault(); options[options.length - 1].focus(); }
+  else if (e.key === "Enter" || e.key === " ") { e.preventDefault(); if (options[idx]) ckSelectChoose(sel, options[idx]); }
+  else if (e.key === "Escape") { e.preventDefault(); ckSelectClose(sel); trigger.focus(); }
+});

data/app/assets/stylesheets/completion_kit/application.css

@@ -1922,6 +1922,13 @@ label.ck-checkbox input {
   cursor: pointer;
 }
 
+.ck-radio-info {
+  width: 16px;
+  height: 16px;
+  color: var(--ck-muted);
+  cursor: help;
+}
+
 .ck-field-row {
   display: flex;
   gap: 1rem;
@@ -1974,6 +1981,117 @@ label.ck-checkbox input {
   color: var(--ck-text);
 }
 
+.ck-select {
+  position: relative;
+}
+
+.ck-select__trigger {
+  width: 100%;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 0.5rem;
+  padding: 0.65rem 0.85rem;
+  border: 1px solid var(--ck-line-strong);
+  border-radius: var(--ck-radius);
+  background: var(--ck-bg);
+  color: var(--ck-text);
+  font-family: var(--ck-mono);
+  font-size: 0.9rem;
+  text-align: left;
+  cursor: pointer;
+  transition: border-color 0.15s, box-shadow 0.15s;
+}
+
+.ck-select__label {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.ck-select__trigger:focus-visible,
+.ck-select.is-open .ck-select__trigger {
+  outline: none;
+  border-color: var(--ck-accent);
+  box-shadow: 0 0 0 3px var(--ck-accent-soft);
+}
+
+.ck-select__chevron {
+  display: inline-flex;
+  flex-shrink: 0;
+  color: var(--ck-muted);
+  transition: transform 0.15s;
+}
+
+.ck-select__chevron svg {
+  width: 1rem;
+  height: 1rem;
+}
+
+.ck-select.is-open .ck-select__chevron {
+  transform: rotate(180deg);
+}
+
+.ck-select__menu {
+  position: absolute;
+  z-index: 30;
+  left: 0;
+  right: 0;
+  top: calc(100% + 0.35rem);
+  margin: 0;
+  padding: 0.3rem;
+  list-style: none;
+  background: var(--ck-surface);
+  border: 1px solid var(--ck-line-strong);
+  border-radius: var(--ck-radius-lg);
+  box-shadow: 0 16px 40px rgba(0, 0, 0, 0.45);
+  max-height: 18rem;
+  overflow-y: auto;
+  font-family: var(--ck-mono);
+}
+
+.ck-select__option {
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+  padding: 0.5rem 0.6rem;
+  border-radius: var(--ck-radius);
+  color: var(--ck-text);
+  font-size: 0.9rem;
+  cursor: pointer;
+}
+
+.ck-select__option:hover,
+.ck-select__option:focus-visible {
+  outline: none;
+  background: var(--ck-surface-hover);
+}
+
+.ck-select__option[aria-selected="true"] {
+  color: var(--ck-accent);
+}
+
+.ck-select__tick {
+  display: inline-flex;
+  flex-shrink: 0;
+  width: 1rem;
+  color: var(--ck-accent);
+  opacity: 0;
+}
+
+.ck-select__tick svg {
+  width: 1rem;
+  height: 1rem;
+}
+
+.ck-select__option[aria-selected="true"] .ck-select__tick {
+  opacity: 1;
+}
+
+.ck-select__text {
+  flex: 1;
+}
+
 .ck-section-title {
   font-size: 1.05rem;
   font-weight: 600;
@@ -3238,7 +3356,8 @@ select.ck-input {
 #ck-tab-metric-groups:checked ~ .ck-api-tabs__nav label[for="ck-tab-metric-groups"],
 #ck-tab-agreements:checked ~ .ck-api-tabs__nav label[for="ck-tab-agreements"],
 #ck-tab-tags:checked ~ .ck-api-tabs__nav label[for="ck-tab-tags"],
-#ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"] {
+#ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"],
+#ck-tab-imports:checked ~ .ck-api-tabs__nav label[for="ck-tab-imports"] {
   color: var(--ck-accent);
   background: var(--ck-surface-soft);
   border-left-color: var(--ck-accent);
@@ -3253,7 +3372,8 @@ select.ck-input {
 #ck-tab-metric-groups:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(7),
 #ck-tab-agreements:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(8),
 #ck-tab-tags:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(9),
-#ck-tab-providers:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(10) {
+#ck-tab-providers:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(10),
+#ck-tab-imports:checked ~ .ck-api-tabs__panels .ck-api-tabs__panel:nth-child(11) {
   display: block;
 }
 
@@ -3295,7 +3415,8 @@ select.ck-input {
   #ck-tab-metric-groups:checked ~ .ck-api-tabs__nav label[for="ck-tab-metric-groups"],
   #ck-tab-agreements:checked ~ .ck-api-tabs__nav label[for="ck-tab-agreements"],
   #ck-tab-tags:checked ~ .ck-api-tabs__nav label[for="ck-tab-tags"],
-  #ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"] {
+  #ck-tab-providers:checked ~ .ck-api-tabs__nav label[for="ck-tab-providers"],
+  #ck-tab-imports:checked ~ .ck-api-tabs__nav label[for="ck-tab-imports"] {
     border-left-color: transparent;
     border-bottom-color: var(--ck-accent);
   }

data/app/controllers/completion_kit/metrics_controller.rb

@@ -68,7 +68,7 @@ module CompletionKit
     end
 
     def create
-      @metric = Metric.new(metric_params)
+      @metric = Metric.new(create_metric_params)
 
       if @metric.save
         redirect_to metric_path(@metric), notice: "Metric was successfully created."
@@ -235,6 +235,15 @@ module CompletionKit
       @metric = Metric.find(params[:id])
     end
 
+    def create_metric_params
+      attrs = metric_params
+      if attrs[:metric_type] == "check"
+        attrs.except(:instruction, :rubric_bands)
+      else
+        attrs.except(:check_config)
+      end
+    end
+
     def metric_params
       permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
         rubric_bands: [:stars, :description],

data/app/controllers/completion_kit/runs_controller.rb

@@ -122,7 +122,7 @@ module CompletionKit
 
     def suggest
       if @run.prompt.nil?
-        redirect_to run_path(@run), alert: "Judge-only runs don't have a prompt to improve."
+        redirect_to run_path(@run), alert: "A run that only scores existing outputs has no prompt to improve."
         return
       end
 

data/app/helpers/completion_kit/application_helper.rb

@@ -169,6 +169,48 @@ module CompletionKit
       end
     end
 
+    CHECK_KIND_LABELS = {
+      "contains" => "Contains a phrase",
+      "not_contains" => "Does not contain a phrase",
+      "equals" => "Equals exactly",
+      "regex" => "Matches a pattern",
+      "valid_json" => "Is valid JSON",
+      "json_path_equals" => "A JSON field equals a value",
+      "length_bounds" => "Length is within a range",
+      "no_refusal" => "Is not a refusal"
+    }.freeze
+
+    CHECK_TARGET_LABELS = {
+      "response_text" => "The response text",
+      "input_data" => "The input row",
+      "json_path" => "A value from the response JSON"
+    }.freeze
+
+    CHECK_FIELD_LABELS = {
+      "value" => "Text to look for",
+      "pattern" => "Pattern",
+      "json_path" => "JSON path",
+      "expected" => "Expected value",
+      "target_path" => "Path into the JSON",
+      "min" => "Shortest allowed",
+      "max" => "Longest allowed",
+      "case_sensitive" => "Case sensitive",
+      "multiline" => "Multiline",
+      "trim" => "Trim whitespace"
+    }.freeze
+
+    def ck_check_kind_label(kind)
+      CHECK_KIND_LABELS.fetch(kind.to_s) { kind.to_s.humanize }
+    end
+
+    def ck_check_target_label(target)
+      CHECK_TARGET_LABELS.fetch(target.to_s) { target.to_s.humanize }
+    end
+
+    def ck_check_field_label(key)
+      CHECK_FIELD_LABELS.fetch(key.to_s) { key.to_s.humanize }
+    end
+
     def ck_result_change_badge(change)
       case change
       when "broke"

data/app/models/completion_kit/run.rb

@@ -30,7 +30,7 @@ module CompletionKit
       display_scoped.select(:id)
     end
 
-    # A judge-only run grades a pre-existing column on the dataset instead of
+    # A scoring-only run grades a pre-existing column on the dataset instead of
     # generating new outputs. No prompt is attached; the response text is read
     # from row[output_column]; no LLM generation happens.
     def judge_only?
@@ -442,7 +442,7 @@ module CompletionKit
         self.name = "#{prompt.name} — v#{prompt.version_number} ##{count}"
       elsif dataset.present?
         count = Run.where(prompt_id: nil, dataset_id: dataset.id).count + 1
-        self.name = "#{dataset.name} — judge-only ##{count}"
+        self.name = "#{dataset.name} scoring ##{count}"
       end
     end
 
@@ -461,7 +461,7 @@ module CompletionKit
       return if prompt.present?
 
       if dataset.nil?
-        errors.add(:dataset_id, "is required for a judge-only run (no prompt)")
+        errors.add(:dataset_id, "is required when scoring existing outputs (no prompt)")
         return
       end
 

data/app/services/completion_kit/mcp_tools/judges.rb

@@ -5,7 +5,7 @@ module CompletionKit
 
       TOOLS = {
         "judges_replay" => {
-          description: "Run the current judge against a dataset (judge-only run). Wraps runs_create with prompt_id omitted and output_column supplied. Re-judges existing dataset outputs so you can compare against human verdicts.",
+          description: "Run the current judge against a dataset (scores existing outputs). Wraps runs_create with prompt_id omitted and output_column supplied. Re-judges existing dataset outputs so you can compare against human verdicts.",
           inputSchema: {
             type: "object",
             properties: {

data/app/services/completion_kit/mcp_tools/prompts.rb

@@ -51,7 +51,7 @@ module CompletionKit
           handler: :publish
         },
         "prompts_suggest_improvement" => {
-          description: "Suggest an improved version of a prompt, grounded in a run's test results and judge feedback. Analyzes the run's responses, scores, and reviews, then returns reasoning plus a rewritten template (preserving {{variables}}) and persists it as a Suggestion. Requires a run that has a prompt (not a judge-only run).",
+          description: "Suggest an improved version of a prompt, grounded in a run's test results and judge feedback. Analyzes the run's responses, scores, and reviews, then returns reasoning plus a rewritten template (preserving {{variables}}) and persists it as a Suggestion. Requires a run that has a prompt (not a scoring-only run).",
           inputSchema: {
             type: "object",
             properties: {run_id: {type: "integer", description: "The run whose results ground the improvement."}},
@@ -107,7 +107,7 @@ module CompletionKit
 
       def self.suggest_improvement(args)
         run = Run.find(args["run_id"])
-        return error_result("Judge-only runs don't have a prompt to improve.") if run.prompt.nil?
+        return error_result("A run that only scores existing outputs has no prompt to improve.") if run.prompt.nil?
 
         result = PromptImprovementService.new(run).suggest
         return error_result("The model didn't return a usable rewrite.") if result["suggested_template"].blank?

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -15,7 +15,7 @@ module CompletionKit
           handler: :get
         },
         "runs_create" => {
-          description: "Create a run. Omit prompt_id and provide output_column for a judge-only run that grades a pre-existing dataset column instead of generating new outputs.",
+          description: "Create a run. Omit prompt_id and provide output_column to score existing outputs by grading a pre-existing dataset column instead of generating new ones.",
           inputSchema: {
             type: "object",
             properties: {

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -20,6 +20,7 @@
   <input type="radio" name="ck-api-tab" id="ck-tab-agreements" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-tags" class="ck-api-tabs__radio">
   <input type="radio" name="ck-api-tab" id="ck-tab-providers" class="ck-api-tabs__radio">
+  <input type="radio" name="ck-api-tab" id="ck-tab-imports" class="ck-api-tabs__radio">
 
   <nav class="ck-api-tabs__nav">
     <label for="ck-tab-mcp" class="ck-api-tabs__label">MCP <span class="ck-api-tabs__count"><%= CompletionKit::McpDispatcher.tool_definitions.size %></span></label>
@@ -32,6 +33,7 @@
     <label for="ck-tab-agreements" class="ck-api-tabs__label">Agreements <span class="ck-api-tabs__count">3</span></label>
     <label for="ck-tab-tags" class="ck-api-tabs__label">Tags <span class="ck-api-tabs__count">5</span></label>
     <label for="ck-tab-providers" class="ck-api-tabs__label">Providers <span class="ck-api-tabs__count">5</span></label>
+    <label for="ck-tab-imports" class="ck-api-tabs__label">Imports <span class="ck-api-tabs__count">1</span></label>
   </nav>
 
   <div class="ck-api-tabs__panels">
@@ -124,7 +126,7 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs</p>
         <p class="ck-meta-copy">Create a new run.</p>
-        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>output_column</code> (judge-only: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>)</p>
+        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>)</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"prompt_id\": 1, \"dataset_id\": 1, \"metric_ids\": [1, 2]}'" %>
       </div>
       <div class="ck-api-endpoint">
@@ -379,6 +381,18 @@
             } %>
     </div>
 
+    <div class="ck-api-tabs__panel">
+      <h2 class="ck-section-title">Imports</h2>
+      <p class="ck-copy">Bring an existing <a href="https://www.promptfoo.dev" class="ck-link">promptfoo</a> config into CompletionKit in one call. Prompts, the test dataset, assert-based metrics, and providers are created where they map cleanly and skipped with a reason where they don't.</p>
+      <div class="ck-api-endpoint">
+        <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/imports/promptfoo</p>
+        <p class="ck-meta-copy">Import a promptfooconfig.yaml. Send the YAML as a <code>config</code> param, or POST the raw YAML as the request body. Returns 201 with a mapping summary, or 422 if the YAML cannot be parsed.</p>
+        <p class="ck-api-params"><strong>Request:</strong>&ensp;<code>config</code> (the YAML text) or a raw YAML request body</p>
+        <p class="ck-api-params"><strong>Response 201:</strong>&ensp;<code>prompts</code>, <code>dataset</code>, <code>metrics</code>, and <code>providers</code>, each listing what was <code>created</code> and what was <code>skipped</code> (with a reason)</p>
+        <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/imports/promptfoo \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/x-yaml\" \\\n  --data-binary @promptfooconfig.yaml" %>
+      </div>
+    </div>
+
   </div>
 </div>
 

data/app/views/completion_kit/metrics/_check_spec.html.erb

@@ -1,16 +1,16 @@
 <% config = local_assigns.fetch(:config) %>
 <dl class="ck-check-spec">
   <div class="ck-check-spec__row">
-    <dt class="ck-check-spec__term">Kind</dt>
-    <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= config["check_kind"] %></code></dd>
+    <dt class="ck-check-spec__term">Rule</dt>
+    <dd class="ck-check-spec__val"><%= ck_check_kind_label(config["check_kind"]) %></dd>
   </div>
   <div class="ck-check-spec__row">
-    <dt class="ck-check-spec__term">Target</dt>
-    <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= config["target"] %></code></dd>
+    <dt class="ck-check-spec__term">Checks</dt>
+    <dd class="ck-check-spec__val"><%= ck_check_target_label(config["target"]) %></dd>
   </div>
   <% config.except("check_kind", "target").each do |key, value| %>
     <div class="ck-check-spec__row">
-      <dt class="ck-check-spec__term"><%= key.humanize %></dt>
+      <dt class="ck-check-spec__term"><%= ck_check_field_label(key) %></dt>
       <dd class="ck-check-spec__val"><code class="ck-check-spec__code"><%= value %></code></dd>
     </div>
   <% end %>

data/app/views/completion_kit/metrics/_form.html.erb

@@ -75,20 +75,22 @@
     <% else %>
       <div class="ck-field" data-ck-metric-type>
         <p class="ck-section-title">Metric type</p>
-        <p class="ck-hint">An LLM judge scores each output on a 1-5 rubric. A deterministic check passes or fails with no model call.</p>
+        <p class="ck-hint">The judge gives each response 1 to 5 stars against your rubric. A check just passes or fails, with no AI.</p>
         <label class="ck-radio">
           <%= form.radio_button :metric_type, "llm_judge", checked: !metric.check? %>
           <span>LLM judge (1-5)</span>
+          <%= heroicon_tag "information-circle", variant: :outline, size: 16, class: "ck-radio-info", "aria-hidden": "true", title: "An AI reads each response and rates it 1 to 5 stars against your rubric, with a written reason. Best for subjective quality: tone, helpfulness, accuracy." %>
         </label>
         <label class="ck-radio">
           <%= form.radio_button :metric_type, "check", checked: metric.check? %>
           <span>Deterministic check</span>
+          <%= heroicon_tag "information-circle", variant: :outline, size: 16, class: "ck-radio-info", "aria-hidden": "true", title: "A rule that passes or fails instantly with no AI and no cost. Best for exact things: valid JSON, contains a phrase, no refusal." %>
         </label>
       </div>
     <% end %>
 
     <% if show_judge %>
-    <div class="ck-field ck-field--spacious" data-ck-metric-editor="llm_judge">
+    <div class="ck-field ck-field--spacious" data-ck-metric-editor="llm_judge" <%= "hidden" if metric.check? %>>
       <p class="ck-section-title">Instruction</p>
       <p class="ck-hint">What should the judge assess? This instruction is sent to the LLM judge when scoring outputs.</p>
       <%= form.text_area :instruction, rows: 8, class: "ck-input ck-input--area", placeholder: "Evaluate whether the output...", **ck_field_aria(form, :instruction) %>
@@ -112,7 +114,7 @@
       <% end %>
     </div>
 
-    <div class="ck-field ck-field--spacious">
+    <div class="ck-field ck-field--spacious" data-ck-metric-editor="llm_judge" <%= "hidden" if metric.check? %>>
       <p class="ck-section-title">Rubric<%= render "completion_kit/metrics/rubric_hint" %></p>
       <p class="ck-hint">What each star rating means for this metric.</p>
 
@@ -155,78 +157,77 @@
 
     <% if show_check %>
     <% check = metric.check_config || {} %>
-    <div class="ck-field ck-field--spacious" data-ck-metric-editor="check">
+    <div class="ck-field ck-field--spacious" data-ck-metric-editor="check" <%= "hidden" unless metric.check? %>>
       <p class="ck-section-title">Check</p>
-      <p class="ck-hint">A deterministic pass/fail rule. Fill only the fields the chosen kind needs.</p>
+      <p class="ck-hint">A pass or fail rule with no AI and no cost. You only fill in the fields the chosen rule needs.</p>
 
       <div class="ck-field">
-        <label class="ck-label" for="metric_check_kind">Check kind</label>
-        <select name="metric[check_config][check_kind]" id="metric_check_kind" class="ck-input">
-          <% CompletionKit::Checks::Registry.kinds.each do |kind| %>
-            <option value="<%= kind %>"<%= " selected" if check["check_kind"] == kind %>><%= kind %></option>
-          <% end %>
-        </select>
+        <p class="ck-label" id="metric_check_kind_label">Rule</p>
+        <%= render "completion_kit/shared/branded_select",
+                   name: "metric[check_config][check_kind]",
+                   options: CompletionKit::Checks::Registry.kinds.map { |kind| [kind, ck_check_kind_label(kind)] },
+                   selected: check["check_kind"] || CompletionKit::Checks::Registry.kinds.first,
+                   labelledby: "metric_check_kind_label" %>
       </div>
 
       <div class="ck-field">
-        <label class="ck-label" for="metric_check_target">Target</label>
-        <select name="metric[check_config][target]" id="metric_check_target" class="ck-input">
-          <% CompletionKit::Checks::TargetResolver::TARGETS.each do |target| %>
-            <option value="<%= target %>"<%= " selected" if check["target"] == target %>><%= target %></option>
-          <% end %>
-        </select>
+        <p class="ck-label" id="metric_check_target_label">What to check</p>
+        <%= render "completion_kit/shared/branded_select",
+                   name: "metric[check_config][target]",
+                   options: CompletionKit::Checks::TargetResolver::TARGETS.map { |target| [target, ck_check_target_label(target)] },
+                   selected: check["target"] || CompletionKit::Checks::TargetResolver::TARGETS.first,
+                   labelledby: "metric_check_target_label" %>
       </div>
 
-      <div class="ck-field">
-        <label class="ck-label" for="metric_check_target_path">Target path</label>
-        <p class="ck-hint">Used when target is json_path, e.g. data.items.0.name.</p>
+      <div class="ck-field" data-ck-check-field="target_path">
+        <label class="ck-label" for="metric_check_target_path">Path into the JSON</label>
+        <p class="ck-hint">Where to look inside the response, like data.items.0.name.</p>
         <input type="text" name="metric[check_config][target_path]" id="metric_check_target_path" class="ck-input" value="<%= check["target_path"] %>">
       </div>
 
-      <div class="ck-field">
-        <label class="ck-label" for="metric_check_value">Value</label>
-        <p class="ck-hint">The substring or exact string for contains, not_contains, or equals.</p>
+      <div class="ck-field" data-ck-check-field="value">
+        <label class="ck-label" for="metric_check_value">Text to look for</label>
         <input type="text" name="metric[check_config][value]" id="metric_check_value" class="ck-input" value="<%= check["value"] %>">
       </div>
 
-      <div class="ck-field">
+      <div class="ck-field" data-ck-check-field="pattern">
         <label class="ck-label" for="metric_check_pattern">Pattern</label>
-        <p class="ck-hint">A regular expression for the regex kind.</p>
+        <p class="ck-hint">A regular expression to match against.</p>
         <input type="text" name="metric[check_config][pattern]" id="metric_check_pattern" class="ck-input" value="<%= check["pattern"] %>">
       </div>
 
-      <div class="ck-field">
+      <div class="ck-field" data-ck-check-field="json_path">
         <label class="ck-label" for="metric_check_json_path">JSON path</label>
-        <p class="ck-hint">Dotted path into parsed JSON for json_path_equals.</p>
+        <p class="ck-hint">The path to the value inside the response, like data.status.</p>
         <input type="text" name="metric[check_config][json_path]" id="metric_check_json_path" class="ck-input" value="<%= check["json_path"] %>">
       </div>
 
-      <div class="ck-field">
-        <label class="ck-label" for="metric_check_expected">Expected</label>
-        <p class="ck-hint">The value the JSON path must equal.</p>
+      <div class="ck-field" data-ck-check-field="expected">
+        <label class="ck-label" for="metric_check_expected">Expected value</label>
+        <p class="ck-hint">The value it should equal.</p>
         <input type="text" name="metric[check_config][expected]" id="metric_check_expected" class="ck-input" value="<%= check["expected"] %>">
       </div>
 
       <div class="ck-field-row">
-        <div class="ck-field">
-          <label class="ck-label" for="metric_check_min">Min length</label>
+        <div class="ck-field" data-ck-check-field="min">
+          <label class="ck-label" for="metric_check_min">Shortest allowed</label>
           <input type="number" name="metric[check_config][min]" id="metric_check_min" class="ck-input" value="<%= check["min"] %>">
         </div>
-        <div class="ck-field">
-          <label class="ck-label" for="metric_check_max">Max length</label>
+        <div class="ck-field" data-ck-check-field="max">
+          <label class="ck-label" for="metric_check_max">Longest allowed</label>
           <input type="number" name="metric[check_config][max]" id="metric_check_max" class="ck-input" value="<%= check["max"] %>">
         </div>
       </div>
 
-      <label class="ck-checkbox">
+      <label class="ck-checkbox" data-ck-check-field="case_sensitive">
         <input type="checkbox" name="metric[check_config][case_sensitive]" value="true"<%= " checked" if check["case_sensitive"] %>>
         <span>Case sensitive</span>
       </label>
-      <label class="ck-checkbox">
+      <label class="ck-checkbox" data-ck-check-field="multiline">
         <input type="checkbox" name="metric[check_config][multiline]" value="true"<%= " checked" if check["multiline"] %>>
         <span>Multiline</span>
       </label>
-      <label class="ck-checkbox">
+      <label class="ck-checkbox" data-ck-check-field="trim">
         <input type="checkbox" name="metric[check_config][trim]" value="true"<%= " checked" if check["trim"] %>>
         <span>Trim whitespace</span>
       </label>

data/app/views/completion_kit/runs/_form.html.erb

@@ -22,7 +22,7 @@
         <%= check_box_tag "run[judge_only]", "1", run.persisted? && run.judge_only?, id: "run_judge_only", class: "ck-checkbox" %>
         <span class="ck-checkbox-label__box" aria-hidden="true"></span>
         <span class="ck-checkbox-label__body">
-          <span class="ck-checkbox-label__text">Judge-only run</span>
+          <span class="ck-checkbox-label__text">Score existing outputs</span>
           <span class="ck-checkbox-label__hint">Grade an existing column on the dataset instead of running a prompt. Roughly half the LLM calls per row.</span>
         </span>
       </label>
@@ -263,7 +263,7 @@ function updateRunForm() {
       }
     } else if (!dataset) {
       if (datasetField) datasetField.className = 'ck-field ck-field--info';
-      if (datasetHint) datasetHint.textContent = 'Judge-only runs need a dataset that supplies the output column.';
+      if (datasetHint) datasetHint.textContent = 'Skip generation and score responses you already have from a dataset column. Works with rubric metrics or deterministic checks.';
     }
   } else {
     valid = prompt !== '';

data/app/views/completion_kit/runs/_row.html.erb

@@ -10,7 +10,7 @@
           <%= link_to run.prompt.name, ck_prompt_path(run.prompt), class: "ck-runs-table__config-link", onclick: "event.stopPropagation();" %>
           <span class="ck-runs-table__version">v<%= run.prompt.version_number %></span>
         <% else %>
-          <span class="ck-runs-table__version">Judge-only</span>
+          <span class="ck-runs-table__version">Scoring only</span>
         <% end %>
         <% if run.dataset %>
           <span class="ck-runs-table__sep">·</span>

data/app/views/completion_kit/runs/_status_header.html.erb

@@ -22,7 +22,7 @@
       <% if run.prompt %>
         <p class="ck-meta-copy"><%= link_to run.prompt.display_name, prompt_path(run.prompt), class: "ck-link" %>&ensp;<span class="ck-chip" style="text-transform: none;"><%= run.prompt.llm_model %></span></p>
       <% else %>
-        <p class="ck-meta-copy">Judge-only run — grading column <code><%= run.output_column.presence || "actual_output" %></code><% if run.dataset %> on <%= link_to run.dataset.name, dataset_path(run.dataset), class: "ck-link" %><% end %></p>
+        <p class="ck-meta-copy">Scoring existing outputs, grading column <code><%= run.output_column.presence || "actual_output" %></code><% if run.dataset %> on <%= link_to run.dataset.name, dataset_path(run.dataset), class: "ck-link" %><% end %></p>
       <% end %>
     </div>
     <%= render "completion_kit/runs/actions", run: run %>

data/app/views/completion_kit/shared/_branded_select.html.erb

@@ -0,0 +1,18 @@
+<% chosen = options.find { |value, _label| value.to_s == selected.to_s } || options.first %>
+<div class="ck-select" data-ck-select>
+  <input type="hidden" name="<%= name %>" value="<%= chosen && chosen.first %>" data-ck-select-value>
+  <button type="button" class="ck-select__trigger" data-ck-select-trigger
+          aria-haspopup="listbox" aria-expanded="false" aria-labelledby="<%= labelledby %>">
+    <span class="ck-select__label" data-ck-select-label><%= chosen && chosen.last %></span>
+    <span class="ck-select__chevron" aria-hidden="true"><%= heroicon_tag "chevron-down" %></span>
+  </button>
+  <ul class="ck-select__menu" role="listbox" hidden data-ck-select-menu aria-labelledby="<%= labelledby %>">
+    <% options.each do |value, label| %>
+      <li class="ck-select__option" role="option" tabindex="-1" data-value="<%= value %>"
+          aria-selected="<%= chosen && chosen.first.to_s == value.to_s ? "true" : "false" %>">
+        <span class="ck-select__tick" aria-hidden="true"><%= heroicon_tag "check" %></span>
+        <span class="ck-select__text" data-ck-select-text><%= label %></span>
+      </li>
+    <% end %>
+  </ul>
+</div>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.20.3"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.20.3
+  version: 0.21.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -419,6 +419,7 @@ files:
 - app/views/completion_kit/runs/index.html.erb
 - app/views/completion_kit/runs/new.html.erb
 - app/views/completion_kit/runs/show.html.erb
+- app/views/completion_kit/shared/_branded_select.html.erb
 - app/views/completion_kit/shared/_settings_nav.html.erb
 - app/views/completion_kit/suggestions/_scoreboard.html.erb
 - app/views/completion_kit/suggestions/_state.html.erb