completion-kit 0.28.14 → 0.28.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3aa50615418395ad99363761cda1ae26f1423659511d52db2252cd41b5cd13d9
-  data.tar.gz: 053ac0c75a8f0eaf921132adf4f6a6dee9f926495a0698700213e143fbc43a59
+  metadata.gz: 73425cba956dc09f3a16e93b78cd3d64ca5f840ccf0c9dbd86bdf99adfafe1a3
+  data.tar.gz: 480da4d4bb8491e780bbe81f97031259f50148efbbd29c8e1ae1ae1d0878fca8
 SHA512:
-  metadata.gz: 9eb01763e01bc3b96d0994a73dbf5a38ecd9f4ea928c133cca98575879eb0088754b256ae540ca2c2df66fcf94608f3de90acf9e88a14d2aa0c12939d8e45840
-  data.tar.gz: bf6992f1fb1a1f19abb7f1b59cde253a23e661a7301f590511328e7c1a662b6ed4199701d5b235beba681e15b6b9056dd80567301d338a15f391f8fa6b837fa0
+  metadata.gz: e26ab2bc7b435f18d89d6a68799c4b7f24ae8d8733e7db89ed309bf79ae4785b2e82d31ab94fc8b838c7a257e62909c5b4ab7ed303d10019c423577fdb4f8d58
+  data.tar.gz: 82a4cffb2ebf3205cba3b09756c6d3e753ab28777f578be0e08cc092e00fa661ec7109ce567e451e2945e8b651a86c593e556afd219f328051dfddbf2894915d

data/app/assets/stylesheets/completion_kit/application.css

@@ -4554,6 +4554,12 @@ table.ck-runs-table {
   border-radius: 0 var(--ck-radius) var(--ck-radius) 0;
 }
 
+.ck-suggest-caveat {
+  margin: 0 0 0.6rem;
+  color: var(--ck-warning);
+  font-size: 0.85rem;
+}
+
 .ck-suggest-reasoning .ck-kicker {
   margin-bottom: 0.5rem;
 }

data/app/controllers/completion_kit/prompts_controller.rb

@@ -9,7 +9,7 @@ module CompletionKit
     
     def show
       @runs = Run.where(prompt_id: @prompt.family_versions.select(:id))
-                 .includes(:prompt, :dataset, responses: :reviews)
+                 .includes(:prompt, :dataset, :tags, responses: :reviews)
                  .order(created_at: :desc)
                  .display_scoped
     end

data/app/helpers/completion_kit/application_helper.rb

@@ -1,5 +1,14 @@
 module CompletionKit
   module ApplicationHelper
+    def ck_markdown(text)
+      return "".html_safe if text.blank?
+
+      inline = text.to_s
+        .gsub(/\*\*(.+?)\*\*/, '<strong>\1</strong>')
+        .gsub(/`([^`]+)`/, '<code>\1</code>')
+      simple_format(inline)
+    end
+
     def ck_runs_display_footer(runs)
       partial = CompletionKit.config.runs_display_footer_partial
       return unless partial

data/app/models/completion_kit/run.rb

@@ -117,7 +117,12 @@ module CompletionKit
     end
 
     def reviews_for_summary
-      @reviews_for_summary ||= Review.where(response_id: responses.select(:id)).to_a
+      @reviews_for_summary ||=
+        if responses.loaded? && responses.all? { |response| response.association(:reviews).loaded? }
+          responses.flat_map(&:reviews)
+        else
+          Review.where(response_id: responses.select(:id)).to_a
+        end
     end
 
     def avg_score

data/app/views/completion_kit/prompts/show.html.erb

@@ -35,7 +35,7 @@
 <section class="ck-card--spaced">
   <div class="ck-prompt-preview__header">
     <p class="ck-kicker">Prompt</p>
-    <% judged_run = @runs.select { |r| r.prompt_id == @prompt.id && r.status == "completed" }.find { |r| r.responses.joins(:reviews).exists? } %>
+    <% judged_run = @runs.select { |r| r.prompt_id == @prompt.id && r.status == "completed" }.find { |r| r.responses.any? { |resp| resp.reviews.any? } } %>
     <% if judged_run %>
       <%= button_to suggest_run_path(judged_run), method: :post, class: ck_button_classes(:light, variant: :outline) + " ck-button--sm", form_class: "inline-block" do %>
         <%= heroicon_tag "sparkles", variant: :outline, class: "ck-magic-icon", "aria-hidden": "true" %>
@@ -48,7 +48,8 @@
   <pre class="ck-code ck-code--dark ck-code--prompt"><%= @prompt.template %></pre>
 </section>
 
-<% versions = @prompt.family_versions.includes(runs: { responses: :reviews }).to_a %>
+<% versions = @prompt.family_versions.to_a %>
+<% runs_by_version = @runs.group_by(&:prompt_id) %>
 <% predecessor_of = versions.index_with { |v| versions.detect { |o| o.version_number < v.version_number } } %>
 <% version_changed = ->(v, pred) { pred && (pred.template != v.template || pred.llm_model != v.llm_model) } %>
 <% if versions.size > 1 %>
@@ -65,7 +66,7 @@
       </thead>
       <tbody>
         <% versions.each do |v| %>
-          <% scoped_runs = v.runs.display_scoped %>
+          <% scoped_runs = runs_by_version[v.id] || [] %>
           <% best_score = scoped_runs.map(&:avg_score).compact.max %>
           <% best_pass_rate = scoped_runs.map(&:check_pass_rate).compact.max %>
           <% pred = predecessor_of[v] %>

data/app/views/completion_kit/suggestions/_state.html.erb

@@ -16,8 +16,8 @@
     <% end %>
 
     <div class="ck-suggest-reasoning">
-      <p class="ck-kicker">Why these changes</p>
-      <div class="ck-suggest-reasoning__body"><%= simple_format(suggestion.reasoning) %></div>
+      <p class="ck-kicker">Reasons for suggested changes</p>
+      <div class="ck-suggest-reasoning__body"><%= ck_markdown(suggestion.reasoning) %></div>
     </div>
 
     <div class="ck-suggest-diff">
@@ -41,6 +41,14 @@
       <pre class="ck-code ck-code--dark"><%= suggestion.suggested_template %></pre>
     </div>
 
+    <% unless suggestion.applied_at? %>
+      <% if !suggestion.validated? %>
+        <p class="ck-suggest-caveat">Couldn't be re-scored against this run's responses, so applying it is unvalidated.</p>
+      <% elsif suggestion.net_negative? %>
+        <% vs = suggestion.validation_summary %>
+        <p class="ck-suggest-caveat">Scored <%= vs["after_avg"] %> on the held-out responses, below your original's <%= vs["before_avg"] %>.</p>
+      <% end %>
+    <% end %>
     <div class="ck-actions">
       <% if suggestion.applied_at? %>
         <span class="ck-chip" style="background: var(--ck-success-soft); color: var(--ck-success);">Applied</span>

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.28.14"
+  VERSION = "0.28.16"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.28.14
+  version: 0.28.16
 platform: ruby
 authors:
 - Damien Bastin