completion-kit 0.5.5 → 0.5.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89a2dd49e9edb75dada51386769cfd7e16f596c3b30c434588864487ab801d5b
-  data.tar.gz: df8d9acfd4ef8aeab73508e7cf9adb2c083c7b9ef3d1c4304e1a3e01723b17e1
+  metadata.gz: 33daa5f930979539f0b757bd5ea85273b9620f90f3de0bf4d976c843cae24af5
+  data.tar.gz: 3387de72acf76fbeac7d79628144ac9cb3782678cf186ad883d1978bceb4daee
 SHA512:
-  metadata.gz: 9b03af38565cb43b7a57747896136817d954b06a89ba4551afb314ca93cd56a70e9d3068271d4f522741efa7d830b106a3a39e05997ea38fc3fb65996741ad08
-  data.tar.gz: be0dad1d98c850854770351edbfdcd7867fbda7b6590634bae2997707a136a94cee066c79fd2ae8a259ea9c01db06a89b7088012a072655c7c627f7c475a801a
+  metadata.gz: 3af6ed91c4c791859b41cddf24078fc0ecec5a79a5aa6ead1ff996c07f9173164693bea12ac223443ae33fcadb1499919734c112a453260f4e9e187cbd6cfd37
+  data.tar.gz: cf05c70a638f3ba9c70657b9c4ab90154af4b5d15e44dbff80b00bd2fc5804e2f3c6313f22fba5283ab36d35adb6d218ec3e1792ee533191c1a7bb66eef59de6

data/README.md

@@ -142,7 +142,7 @@ Only one mode can be active.
 
 ## Concepts
 
-- **Prompt.** A versioned template with `{{variable}}` placeholders. Publishing freezes the template; editing a published prompt creates a new version.
+- **Prompt.** A versioned template with `{{variable}}` placeholders. Editing a prompt that's already been run creates a new version, so earlier results stay reproducible.
 - **Dataset.** A CSV of real inputs. Each row becomes one test case.
 - **Run.** One execution of a prompt against a dataset. Captures every input (model, temperature, metrics) and stores all outputs and scores.
 - **Response.** The model's output for one dataset row, with reviews attached.

data/app/assets/javascripts/completion_kit/application.js

@@ -105,11 +105,16 @@ document.addEventListener("mouseout", function(e) {
 });
 
 var ckRefreshing = false;
+function ckSetRefreshButtonsBusy(busy) {
+  document.querySelectorAll('.ck-icon-btn[title="Refresh models"]').forEach(function(btn) {
+    btn.classList.toggle('ck-icon-btn--spinning', busy);
+    btn.disabled = busy;
+  });
+}
 function ckRefreshModels() {
   if (ckRefreshing) return;
   ckRefreshing = true;
-  var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');
-  if (btn) btn.classList.add('ck-icon-btn--spinning');
+  ckSetRefreshButtonsBusy(true);
   ckUpdateRefreshProgress();
   var csrfToken = document.querySelector('meta[name="csrf-token"]').getAttribute("content");
   fetch("/completion_kit/refresh_models", {
@@ -149,8 +154,7 @@ document.addEventListener("turbo:before-stream-render", function(event) {
   }
   if (target === "prompt_llm_model" || target === "run_judge_model") {
     ckRefreshing = false;
-    var btn = document.querySelector('.ck-icon-btn[title="Refresh models"]');
-    if (btn) btn.classList.remove('ck-icon-btn--spinning');
+    ckSetRefreshButtonsBusy(false);
     var status = document.getElementById('refresh-status');
     if (status) { status.textContent = 'Models updated.'; setTimeout(function() { status.textContent = ' '; }, 3000); }
   }

data/app/assets/stylesheets/completion_kit/application.css

@@ -1164,6 +1164,17 @@ tr:hover .ck-chip--publish {
   cursor: help;
 }
 
+.ck-model-table tr.ck-model-table__section td {
+  padding: 0.85rem 0 0.3rem;
+  font-family: var(--ck-mono);
+  font-size: 0.66rem;
+  font-weight: 600;
+  letter-spacing: 0.16em;
+  text-transform: uppercase;
+  color: var(--ck-dim);
+}
+.ck-model-table tr.ck-model-table__section:first-child td { padding-top: 0; }
+
 .ck-model-list__summary {
   display: flex;
   align-items: center;
@@ -1278,6 +1289,102 @@ tr:hover .ck-chip--publish {
   max-width: 36rem;
 }
 
+.ck-version-note {
+  margin: 0 0 1rem;
+  max-width: 36rem;
+  font-family: var(--ck-sans);
+  font-size: 0.85rem;
+  line-height: 1.5;
+  color: var(--ck-muted);
+}
+
+.ck-version-change {
+  margin: 0.5rem 0 0;
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+  flex-wrap: wrap;
+  font-family: var(--ck-mono);
+  font-size: 0.8rem;
+}
+.ck-version-change__label {
+  font-size: 0.66rem;
+  font-weight: 600;
+  letter-spacing: 0.12em;
+  text-transform: uppercase;
+  color: var(--ck-dim);
+}
+.ck-version-change__old {
+  padding: 0.15em 0.5em;
+  border-radius: var(--ck-radius);
+  background: var(--ck-danger-soft);
+  color: var(--ck-danger);
+  text-decoration: line-through;
+}
+.ck-version-change__arrow { color: var(--ck-dim); }
+.ck-version-change__new {
+  padding: 0.15em 0.5em;
+  border-radius: var(--ck-radius);
+  background: var(--ck-success-soft);
+  color: var(--ck-success);
+}
+
+.ck-modal__body .ck-version-change + .ck-suggest-diff { margin-top: 0.9rem; }
+
+.ck-cell-link {
+  background: none;
+  border: none;
+  padding: 0;
+  font-family: var(--ck-mono);
+  font-size: 0.8rem;
+  color: var(--ck-accent);
+  cursor: pointer;
+  text-decoration: underline;
+  text-underline-offset: 2px;
+}
+.ck-cell-link:hover { color: var(--ck-accent-hover); }
+.ck-cell-link--delta {
+  font-size: 1rem;
+  font-weight: 700;
+  text-decoration: none;
+  color: var(--ck-dim);
+}
+.ck-cell-link--delta:hover { color: var(--ck-accent); }
+
+/* Version-history table: cell that pairs the version label + publish control (left) with the diff trigger (right) */
+.ck-version-cell {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  gap: 0.75rem;
+}
+.ck-version-cell__label { display: flex; align-items: center; gap: 0.5rem; }
+
+/* the version currently being viewed: faint accent wash (like the model chips)
+   plus a bright accent border drawn on the cells (Safari can't box-shadow a <tr>) */
+.ck-results-table tbody tr.ck-results-table__row--active,
+.ck-results-table tbody tr.ck-results-table__row--active:hover {
+  background: var(--ck-accent-soft);
+}
+.ck-results-table tbody tr.ck-results-table__row--active td {
+  box-shadow: inset 0 2px 0 var(--ck-accent), inset 0 -2px 0 var(--ck-accent);
+}
+.ck-results-table tbody tr.ck-results-table__row--active td:first-child {
+  box-shadow: inset 0 2px 0 var(--ck-accent), inset 0 -2px 0 var(--ck-accent), inset 2px 0 0 var(--ck-accent);
+}
+.ck-results-table tbody tr.ck-results-table__row--active td:last-child {
+  box-shadow: inset 0 2px 0 var(--ck-accent), inset 0 -2px 0 var(--ck-accent), inset -2px 0 0 var(--ck-accent);
+}
+/* when it's the last row, round its bottom corners so the border isn't clipped
+   by the table's rounded container */
+.ck-results-table tbody tr.ck-results-table__row--active:last-child td:first-child {
+  border-bottom-left-radius: calc(var(--ck-radius-lg) - 1px);
+}
+.ck-results-table tbody tr.ck-results-table__row--active:last-child td:last-child {
+  border-bottom-right-radius: calc(var(--ck-radius-lg) - 1px);
+}
+.ck-results-table tbody tr.ck-results-table__row--active strong { color: var(--ck-accent-hover); }
+
 .ck-field-hint {
   font-family: var(--ck-sans);
   font-size: 0.8rem;
@@ -1659,30 +1766,54 @@ tr:hover .ck-chip--publish {
   color: var(--ck-text);
 }
 
+@keyframes ck-flash-in {
+  from { opacity: 0; transform: translateY(-4px); }
+  to   { opacity: 1; transform: translateY(0); }
+}
+
 .ck-flash {
-  margin-bottom: 1rem;
-  padding: 0.75rem 1rem;
-  border-radius: var(--ck-radius);
+  margin-bottom: 1.25rem;
+  padding: 0.8rem 1rem;
   border: 1px solid transparent;
+  border-radius: var(--ck-radius);
   font-family: var(--ck-sans);
-  font-size: 0.95rem;
+  font-size: 0.9rem;
+  line-height: 1.6;
+  animation: ck-flash-in 0.22s ease both;
 }
+.ck-flash--notice { background: rgba(45, 212, 168, 0.08); border-color: rgba(45, 212, 168, 0.32); }
+.ck-flash--alert  { background: rgba(248, 113, 113, 0.08); border-color: rgba(248, 113, 113, 0.32); }
 
-.ck-flash--notice {
-  border-color: rgba(34, 197, 94, 0.3);
-  background: var(--ck-success-soft);
-  color: var(--ck-success);
+/* layout flash: a mono status tag (the engine's kicker treatment) inline before the message */
+.ck-flash__label {
+  margin-right: 0.6rem;
+  font-family: var(--ck-mono);
+  font-size: 0.7rem;
+  font-weight: 600;
+  letter-spacing: 0.14em;
+  text-transform: uppercase;
+  white-space: nowrap;
 }
-
-.ck-flash--alert {
-  border-color: rgba(239, 68, 68, 0.3);
-  background: var(--ck-danger-soft);
-  color: var(--ck-danger);
+.ck-flash__label::before {
+  content: "";
+  display: inline-block;
+  width: 6px;
+  height: 6px;
+  margin-right: 0.45rem;
+  border-radius: 50%;
+  background: currentColor;
+  box-shadow: 0 0 7px currentColor;
+  vertical-align: 0.12em;
 }
+.ck-flash--notice .ck-flash__label { color: var(--ck-success); }
+.ck-flash--alert  .ck-flash__label { color: var(--ck-danger); }
+.ck-flash__body { color: var(--ck-text); }
 
-.ck-flash__title {
-  margin: 0;
-  font-weight: 700;
+/* form-error summary inside .ck-flash--alert: title sentence + ck-error-list */
+.ck-flash__title { margin: 0; font-weight: 600; color: var(--ck-danger); }
+
+@media (prefers-reduced-motion: reduce) {
+  .ck-flash { animation: none; }
 }
 
 .ck-banner {
@@ -2012,6 +2143,10 @@ select.ck-input {
   box-shadow: 0 30px 80px rgba(0, 0, 0, 0.55);
   overflow: hidden;
 }
+/* the panel is the dialog's autofocus target (so the close button isn't auto-focused);
+   it's a container, not a control, so don't show a ring on it */
+.ck-modal__panel:focus,
+.ck-modal__panel:focus-visible { outline: none; }
 
 .ck-modal__header {
   display: flex;
@@ -2048,9 +2183,8 @@ select.ck-input {
 
 .ck-modal__close {
   appearance: none;
+  position: relative;
   display: inline-flex;
-  align-items: center;
-  justify-content: center;
   width: 28px;
   height: 28px;
   padding: 0;
@@ -2059,12 +2193,25 @@ select.ck-input {
   outline: none;
   background: transparent;
   color: var(--ck-dim);
-  font-family: var(--ck-sans);
-  font-size: 1.5rem;
-  line-height: 1;
+  font-size: 0;
   cursor: pointer;
   transition: color 0.15s, background 0.15s;
 }
+/* the "×" is drawn as two crossed bars centred on the button, so it (and the
+   focus ring around the button) sit dead centre regardless of font metrics */
+.ck-modal__close::before,
+.ck-modal__close::after {
+  content: "";
+  position: absolute;
+  top: 50%;
+  left: 50%;
+  width: 13px;
+  height: 1.6px;
+  border-radius: 1px;
+  background: currentColor;
+}
+.ck-modal__close::before { transform: translate(-50%, -50%) rotate(45deg); }
+.ck-modal__close::after  { transform: translate(-50%, -50%) rotate(-45deg); }
 
 .ck-modal__close:hover {
   color: var(--ck-text);
@@ -2083,6 +2230,8 @@ select.ck-input {
   overflow: auto;
   padding: 0 1.5rem;
 }
+/* no footer? then the body needs its own bottom padding */
+.ck-modal__body:last-child { padding-bottom: 1.5rem; }
 
 .ck-modal__footer {
   display: flex;
@@ -2603,9 +2752,15 @@ select.ck-input {
   }
 }
 
+/* the metrics field stacks several sub-sections (hint, groups, divider, tag
+   filter, checkboxes) — give it more vertical breathing room than a plain field */
+#metrics-field {
+  gap: 0.85rem;
+}
+
 .ck-metric-groups {
   display: grid;
-  gap: 0.4rem;
+  gap: 0.55rem;
   margin-bottom: 0.5rem;
 }
 
@@ -2621,7 +2776,7 @@ select.ck-input {
 .ck-metric-groups__row {
   display: flex;
   flex-wrap: wrap;
-  gap: 0.4rem;
+  gap: 0.5rem;
 }
 
 .ck-metric-group-pill {
@@ -2878,6 +3033,29 @@ select.ck-input {
   color: var(--ck-dim);
 }
 
+.ck-prompts-table__desc {
+  margin: 0.3rem 0 0;
+  font-family: var(--ck-sans);
+  font-size: 0.82rem;
+  font-weight: 400;
+  line-height: 1.45;
+  color: var(--ck-muted);
+  white-space: normal;
+  max-width: 42ch;
+}
+
+.ck-endpoint--compact {
+  margin-top: 0.45rem;
+  max-width: 26rem;
+}
+.ck-endpoint--compact .ck-endpoint__url {
+  min-width: 0;
+  white-space: nowrap;
+  overflow: hidden;
+  text-overflow: ellipsis;
+}
+.ck-endpoint--compact .ck-icon-btn { flex-shrink: 0; }
+
 .ck-clamp-2 {
   display: -webkit-box;
   -webkit-line-clamp: 2;

data/app/controllers/completion_kit/prompts_controller.rb

@@ -50,7 +50,7 @@ module CompletionKit
 
     def publish
       @prompt.publish!
-      redirect_to prompt_path(@prompt), notice: "#{@prompt.display_name} is now the current version."
+      redirect_to prompt_path(@prompt), notice: "#{@prompt.display_name} is now the published version."
     end
 
     private

data/app/controllers/completion_kit/runs_controller.rb

@@ -28,6 +28,11 @@ module CompletionKit
 
     def new
       @run = Run.new(prompt_id: params[:prompt_id])
+      prompt = Prompt.find_by(id: @run.prompt_id)
+      if prompt
+        last_run = Run.where(prompt_id: prompt.family_versions.ids).order(created_at: :desc).first
+        @run.tag_names = last_run.tag_names if last_run
+      end
     end
 
     def edit
@@ -79,6 +84,7 @@ module CompletionKit
         dataset_id: @run.dataset_id,
         judge_model: @run.judge_model,
         temperature: @run.temperature,
+        tag_names: @run.tag_names,
         status: "pending"
       )
       new_run.replace_metrics!(@run.metric_ids)

data/app/helpers/completion_kit/application_helper.rb

@@ -72,6 +72,47 @@ module CompletionKit
       CompletionKit::ProviderCredential::PROVIDER_LABELS[provider.to_s] || provider.to_s.titleize
     end
 
+    def ck_masked_token(token)
+      return "YOUR_TOKEN" if token.blank?
+      return "••••••••" if token.length < 12
+      "#{token[0..3]}#{'•' * [token.length - 8, 4].max}#{token[-4..]}"
+    end
+
+    OPENAI_MODEL_FAMILY_ORDER = ["GPT-5", "GPT-4", "o-series", "GPT-3.5", "GPT-OSS", "Other"].freeze
+
+    def ck_openai_model_family(model_id)
+      id = model_id.to_s
+      return "GPT-5" if id.match?(/\Agpt-5/i)
+      return "GPT-4" if id.match?(/\Agpt-4/i)
+      return "GPT-3.5" if id.match?(/\Agpt-3/i)
+      return "GPT-OSS" if id.match?(/\Agpt-oss/i)
+      return "o-series" if id.match?(/\Ao\d/i)
+      "Other"
+    end
+
+    # Groups a provider's models for the models-card table, mirroring how the
+    # dropdown sub-groups: OpenRouter clusters by upstream vendor (the part
+    # before "/"); OpenAI clusters by family; everyone else stays flat. Returns
+    # [[section_label_or_nil, [models]], ...]. A single section collapses to a
+    # nil label so we don't render a redundant header.
+    def ck_model_table_sections(models)
+      models = models.to_a
+      sections =
+        case models.first&.provider
+        when "openrouter"
+          models.group_by { |m| m.model_id.to_s.split("/", 2).first.delete_prefix("~") }
+                .sort_by { |label, _| label }
+        when "openai"
+          grouped = models.group_by { |m| ck_openai_model_family(m.model_id) }
+          ordered = OPENAI_MODEL_FAMILY_ORDER.filter_map { |label| [label, grouped[label]] if grouped[label] }
+          extras = (grouped.keys - OPENAI_MODEL_FAMILY_ORDER).sort.map { |label| [label, grouped[label]] }
+          ordered + extras
+        else
+          [[nil, models]]
+        end
+      sections.size <= 1 ? [[nil, models]] : sections
+    end
+
     def ck_model_option_label(model)
       return "#{model[:name]} (?)" if model.key?(:judging_confirmed) && !model[:judging_confirmed]
       model[:name]
@@ -85,20 +126,33 @@ module CompletionKit
         end
       end
 
-      groups = models.group_by do |m|
-        if m[:provider] == "openrouter"
-          upstream = m[:id].to_s.split("/", 2).first
-          "OpenRouter — #{upstream}"
-        else
-          ck_provider_label(m[:provider])
-        end
-      end
-
-      ordered_keys = groups.keys.sort_by { |label| [label.start_with?("OpenRouter") ? 1 : 0, label] }
+      groups = models.group_by { |m| ck_model_optgroup_label(m) }
+      ordered_keys = groups.keys.sort_by { |label| ck_model_optgroup_sort_key(label) }
       grouped = ordered_keys.map { |label| [label, groups[label].map { |m| [ck_model_option_label(m), m[:id]] }] }
       grouped_options_for_select(grouped, selected)
     end
 
+    # Optgroup label for the model select — mirrors the provider models table:
+    # OpenRouter splits by upstream vendor, OpenAI splits by family, everyone
+    # else is a single group.
+    def ck_model_optgroup_label(model)
+      case model[:provider]
+      when "openrouter" then "OpenRouter — #{model[:id].to_s.split("/", 2).first.delete_prefix("~")}"
+      when "openai"     then "OpenAI — #{ck_openai_model_family(model[:id])}"
+      else ck_provider_label(model[:provider])
+      end
+    end
+
+    def ck_model_optgroup_sort_key(label)
+      if label.start_with?("OpenAI — ")
+        [0, OPENAI_MODEL_FAMILY_ORDER.index(label.delete_prefix("OpenAI — ")), label]
+      elsif label.start_with?("OpenRouter")
+        [2, 0, label]
+      else
+        [1, 0, label]
+      end
+    end
+
     def ck_model_options_html(scope)
       models = CompletionKit::ApiConfig.available_models(scope: scope)
       return "" if models.empty?

data/app/models/completion_kit/provider_credential.rb

@@ -47,6 +47,14 @@ module CompletionKit
       false
     end
 
+    def model_count
+      Model.where(provider: provider).active.count
+    end
+
+    def self.discovery_in_progress?
+      where(discovery_status: "discovering").exists?
+    end
+
     def prompt_count
       model_ids = Model.where(provider: provider).pluck(:model_id)
       return 0 if model_ids.empty?

data/app/services/completion_kit/worker_health.rb

@@ -1,6 +1,9 @@
 module CompletionKit
   class WorkerHealth
-    HEARTBEAT_THRESHOLD = 30.seconds
+    # SolidQueue workers heartbeat every SolidQueue.process_heartbeat_interval
+    # (60s by default), so a 30s window flagged healthy workers as down between
+    # beats. Allow two missed heartbeats before we say the worker is gone.
+    HEARTBEAT_THRESHOLD = 2.minutes
 
     def self.healthy?
       return true unless defined?(::SolidQueue::Process)

data/app/views/completion_kit/api_reference/_authentication.html.erb

@@ -0,0 +1,11 @@
+<% if token.present? %>
+  <% masked = ck_masked_token(token) %>
+  <div style="display: flex; align-items: center; gap: 0.4rem; margin: 0.5rem 0;">
+    <button type="button" class="ck-chip ck-token-toggle" onclick="ckToggleToken(this)" data-masked="<%= masked %>" data-real="<%= token %>" aria-label="Reveal API token" aria-pressed="false" style="cursor: pointer;"><%= masked %></button>
+    <button type="button" class="ck-icon-btn" title="Copy token" aria-label="Copy API token" onclick="var b=this;navigator.clipboard.writeText('<%= token %>').then(function(){b.classList.add('ck-api-copy--done');setTimeout(function(){b.classList.remove('ck-api-copy--done')},1500)})">
+      <%= heroicon_tag "clipboard-document", variant: :outline, size: 14, "aria-hidden": "true" %>
+    </button>
+  </div>
+<% else %>
+  <p class="ck-meta-copy">No API token configured. Set <code>COMPLETION_KIT_API_TOKEN</code> to enable API access.</p>
+<% end %>