completion-kit 0.28.16 → 0.28.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73425cba956dc09f3a16e93b78cd3d64ca5f840ccf0c9dbd86bdf99adfafe1a3
-  data.tar.gz: 480da4d4bb8491e780bbe81f97031259f50148efbbd29c8e1ae1ae1d0878fca8
+  metadata.gz: a06bbc49a6b929962197f8ff7fe9cd5e1fa4561fe2a7074e4028ac867472cf61
+  data.tar.gz: 6615270452a2c991cce7ef84aa5267e22f014e3039dd59cd7ebeb1f0ba8cb90c
 SHA512:
-  metadata.gz: e26ab2bc7b435f18d89d6a68799c4b7f24ae8d8733e7db89ed309bf79ae4785b2e82d31ab94fc8b838c7a257e62909c5b4ab7ed303d10019c423577fdb4f8d58
-  data.tar.gz: 82a4cffb2ebf3205cba3b09756c6d3e753ab28777f578be0e08cc092e00fa661ec7109ce567e451e2945e8b651a86c593e556afd219f328051dfddbf2894915d
+  metadata.gz: 9be6cc2b9f3522f622c9b6d503e391b3007d70e156ea12f862e4cd8fd91a20a3a9c01691a8910a48bdaedd2d9dea86766051ea8bf8172aac7f0882f5368e257c
+  data.tar.gz: b87f4a046822e9b52b2edd823f443894d5f106cc81f1e6470fb5f7d4d4764a497ac23f3795201e63819971d3890c235eb3a297e9b689d2781479cdfc005f89b1

data/app/assets/stylesheets/completion_kit/application.css

@@ -4039,11 +4039,12 @@ select.ck-input {
 .ck-source-chip--past {
   color: var(--ck-dim);
 }
-.ck-suggestions-table th:nth-child(1), .ck-suggestions-table td:nth-child(1) { width: 16rem; white-space: nowrap; }
-.ck-suggestions-table th:nth-child(2), .ck-suggestions-table td:nth-child(2) { width: auto; }
-.ck-suggestions-table th:nth-child(3), .ck-suggestions-table td:nth-child(3) { width: 6rem; white-space: nowrap; }
-.ck-suggestions-table th:nth-child(4), .ck-suggestions-table td:nth-child(4) { width: 9rem; white-space: nowrap; }
-.ck-suggestions-table th:nth-child(5), .ck-suggestions-table td:nth-child(5) { width: 3rem; }
+.ck-suggestions-table th:nth-child(1), .ck-suggestions-table td:nth-child(1) { width: 14rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+.ck-suggestions-table th:nth-child(2), .ck-suggestions-table td:nth-child(2) { width: 5rem; white-space: nowrap; }
+.ck-suggestions-table th:nth-child(3), .ck-suggestions-table td:nth-child(3) { width: auto; }
+.ck-suggestions-table th:nth-child(4), .ck-suggestions-table td:nth-child(4) { width: 6rem; white-space: nowrap; }
+.ck-suggestions-table th:nth-child(5), .ck-suggestions-table td:nth-child(5) { width: 8rem; white-space: nowrap; }
+.ck-suggestions-table th:nth-child(6), .ck-suggestions-table td:nth-child(6) { width: 3rem; }
 
 .ck-metrics-table__groups {
   display: flex;
@@ -4560,6 +4561,37 @@ table.ck-runs-table {
   font-size: 0.85rem;
 }
 
+.ck-suggest-untested {
+  margin-bottom: 16px;
+  padding-bottom: 14px;
+  border-bottom: 1px solid var(--ck-line);
+}
+
+.ck-suggest-untested__headline {
+  margin: 0 0 6px;
+  font-size: 0.95rem;
+  color: var(--ck-text);
+}
+
+.ck-suggest-untested__copy {
+  margin: 0;
+  font-size: 0.85rem;
+  line-height: 1.6;
+  color: var(--ck-muted);
+  max-width: 66ch;
+}
+
+.ck-suggest-decision {
+  margin-top: 1.5rem;
+}
+
+.ck-suggest-decision__note {
+  margin: 0 0 0.9rem;
+  font-size: 0.8rem;
+  color: var(--ck-muted);
+  max-width: 66ch;
+}
+
 .ck-suggest-reasoning .ck-kicker {
   margin-bottom: 0.5rem;
 }

data/app/helpers/completion_kit/application_helper.rb

@@ -9,6 +9,16 @@ module CompletionKit
       simple_format(inline)
     end
 
+    def ck_strip_markdown(text)
+      text.to_s
+        .gsub(/\*\*(.+?)\*\*/, '\1')
+        .gsub(/`([^`]+)`/, '\1')
+        .gsub(/[#>]+/, " ")
+        .gsub(/(?:^|\s)[-*•]\s+/, " ")
+        .gsub(/\s+/, " ")
+        .strip
+    end
+
     def ck_runs_display_footer(runs)
       partial = CompletionKit.config.runs_display_footer_partial
       return unless partial

data/app/views/completion_kit/prompts/show.html.erb

@@ -153,14 +153,15 @@
 
 <%= ck_runs_display_footer(@runs) %>
 
-<% suggestions = CompletionKit::Suggestion.where(prompt_id: @prompt.family_versions.select(:id)).order(created_at: :desc) %>
+<% suggestions = CompletionKit::Suggestion.where(prompt_id: @prompt.family_versions.select(:id)).includes(:run, :prompt).order(created_at: :desc) %>
 <% if suggestions.any? %>
   <section class="ck-card--spaced">
-    <p class="ck-kicker">Suggestions</p>
+    <p class="ck-kicker">Suggested improvements</p>
     <table class="ck-results-table ck-suggestions-table" style="margin-top: 0.5rem;">
       <thead>
         <tr>
           <th>Run</th>
+          <th>Version</th>
           <th>Reasoning</th>
           <th>Applied</th>
           <th>When</th>
@@ -171,7 +172,8 @@
         <% suggestions.each do |s| %>
           <tr onclick="window.location='<%= suggestion_path(s, from: "prompt") %>'" style="cursor: pointer;">
             <td><%= link_to suggestion_path(s, from: "prompt"), class: "ck-record-name" do %><strong><%= s.run.name %></strong><% end %></td>
-            <td class="ck-meta-copy"><%= truncate(s.reasoning.to_s, length: 100) %></td>
+            <td class="ck-meta-copy"><%= s.prompt.version_label %></td>
+            <td class="ck-meta-copy"><%= truncate(ck_strip_markdown(s.reasoning), length: 100) %></td>
             <td><%= s.applied_at? ? content_tag(:span, "Applied", class: "ck-chip", style: "background: var(--ck-success-soft); color: var(--ck-success);") : "&mdash;".html_safe %></td>
             <td class="ck-meta-copy"><time data-relative-time datetime="<%= s.created_at.utc.iso8601 %>"><%= time_ago_in_words(s.created_at) %> ago</time></td>
             <td class="ck-results-table__arrow" aria-hidden="true">&rarr;</td>

data/app/views/completion_kit/suggestions/_scoreboard.html.erb

@@ -1,16 +1,12 @@
 <% s = summary %>
 <div class="ck-scoreboard">
-  <% if s["after_avg"] && s["before_avg"] %>
-    <p class="ck-scoreboard__headline">Scored <strong><%= s["after_avg"] %></strong> across <%= pluralize(s["tested"], "held-out response") %> <span class="ck-scoreboard__was">was <%= s["before_avg"] %></span></p>
-    <ul class="ck-scoreboard__tally">
-      <li class="ck-scoreboard__stat ck-scoreboard__stat--fix">Improved <strong><%= s["improved"] %></strong></li>
-      <li class="ck-scoreboard__stat ck-scoreboard__stat--keep">Held <strong><%= s["unchanged"] %></strong></li>
-      <li class="ck-scoreboard__stat ck-scoreboard__stat--break">Regressed <strong><%= s["regressed"] %></strong></li>
-    </ul>
-  <% else %>
-    <p class="ck-scoreboard__headline">Couldn't re-score this rewrite against the run's responses.</p>
-  <% end %>
+  <p class="ck-scoreboard__headline">We re-ran this wording on <strong><%= s["tested"] %></strong> of the run's responses. It scored <strong><%= s["after_avg"] %></strong> now, <span class="ck-scoreboard__was"><%= s["before_avg"] %> before</span></p>
+  <ul class="ck-scoreboard__tally">
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--fix">Better on <strong><%= s["improved"] %></strong></li>
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--keep">Same on <strong><%= s["unchanged"] %></strong></li>
+    <li class="ck-scoreboard__stat ck-scoreboard__stat--break">Worse on <strong><%= s["regressed"] %></strong></li>
+  </ul>
   <% if s["capped"] %>
-    <p class="ck-scoreboard__note">Tested against this run's 30 most recent responses.</p>
+    <p class="ck-scoreboard__note">Based on this run's 30 most recent responses.</p>
   <% end %>
 </div>

data/app/views/completion_kit/suggestions/_state.html.erb

@@ -11,15 +11,15 @@
       <%= button_to "Try again", suggest_run_path(run), method: :post, class: ck_button_classes(:light, variant: :outline), form_class: "inline-block" %>
     </div>
   <% else %>
-    <% if suggestion.validation_summary.present? %>
+    <% if suggestion.validated? %>
       <%= render "completion_kit/suggestions/scoreboard", summary: suggestion.validation_summary %>
+    <% else %>
+      <div class="ck-suggest-untested">
+        <p class="ck-suggest-untested__headline">We couldn't test this rewrite</p>
+        <p class="ck-suggest-untested__copy">Testing a rewrite means re-running it on this run's responses and scoring the results. That didn't return a score this time, usually because the run has no AI-judge scores to compare against. So there's no evidence yet that it beats your current prompt.</p>
+      </div>
     <% end %>
 
-    <div class="ck-suggest-reasoning">
-      <p class="ck-kicker">Reasons for suggested changes</p>
-      <div class="ck-suggest-reasoning__body"><%= ck_markdown(suggestion.reasoning) %></div>
-    </div>
-
     <div class="ck-suggest-diff">
       <div class="ck-suggest-diff__pane">
         <div class="ck-suggest-diff__header">
@@ -36,28 +36,30 @@
       </div>
     </div>
 
-    <div class="ck-suggest-full">
-      <p class="ck-kicker">Full suggested prompt</p>
-      <pre class="ck-code ck-code--dark"><%= suggestion.suggested_template %></pre>
+    <div class="ck-suggest-reasoning">
+      <p class="ck-kicker">Reasons for suggested changes</p>
+      <div class="ck-suggest-reasoning__body"><%= ck_markdown(suggestion.reasoning) %></div>
     </div>
 
-    <% unless suggestion.applied_at? %>
-      <% if !suggestion.validated? %>
-        <p class="ck-suggest-caveat">Couldn't be re-scored against this run's responses, so applying it is unvalidated.</p>
-      <% elsif suggestion.net_negative? %>
-        <% vs = suggestion.validation_summary %>
-        <p class="ck-suggest-caveat">Scored <%= vs["after_avg"] %> on the held-out responses, below your original's <%= vs["before_avg"] %>.</p>
-      <% end %>
-    <% end %>
-    <div class="ck-actions">
+    <div class="ck-suggest-decision">
       <% if suggestion.applied_at? %>
-        <span class="ck-chip" style="background: var(--ck-success-soft); color: var(--ck-success);">Applied</span>
-      <% elsif !suggestion.validated? %>
-        <%= button_to "Apply anyway", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:light, variant: :outline), form: { class: "inline-block", data: { turbo_confirm: "This rewrite couldn't be re-scored against the run's responses. Apply it anyway?" } } %>
-      <% elsif suggestion.net_negative? %>
-        <%= button_to "Apply anyway", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:light, variant: :outline), form: { class: "inline-block", data: { turbo_confirm: "This rewrite scored lower than the original on the held-out responses. Apply it anyway?" } } %>
+        <div class="ck-actions">
+          <span class="ck-chip" style="background: var(--ck-success-soft); color: var(--ck-success);">Applied</span>
+        </div>
       <% else %>
-        <%= button_to "Apply suggestion", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:dark), form_class: "inline-block" %>
+        <% if suggestion.validated? && suggestion.net_negative? %>
+          <p class="ck-suggest-caveat">This wording scored lower than your current prompt on the responses we tested. Applying it will most likely make results worse.</p>
+        <% end %>
+        <p class="ck-suggest-decision__note">Applying publishes this wording as a new version of your prompt. Your current version stays exactly as it is.</p>
+        <div class="ck-actions">
+          <% if !suggestion.validated? %>
+            <%= button_to "Apply anyway", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:light, variant: :outline), form: { class: "inline-block", data: { turbo_confirm: "We couldn't test this rewrite, so there's no evidence it beats your current prompt. Apply it anyway?" } } %>
+          <% elsif suggestion.net_negative? %>
+            <%= button_to "Apply anyway", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:light, variant: :outline), form: { class: "inline-block", data: { turbo_confirm: "This rewrite scored lower than your current prompt on the responses we tested. Apply it anyway?" } } %>
+          <% else %>
+            <%= button_to "Apply suggestion", apply_suggestion_path(suggestion), method: :post, class: ck_button_classes(:dark), form_class: "inline-block" %>
+          <% end %>
+        </div>
       <% end %>
     </div>
   <% end %>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.16"
+  VERSION = "0.28.17"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.16
+  version: 0.28.17
 platform: ruby
 authors:
 - Damien Bastin