completion-kit 0.23.0 → 0.24.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3952f21a923b362bd293e0cf32cc358b2e635e07124d9b4696e9c8a153b344e
-  data.tar.gz: 2c669f474bbb38dc79b5320b8323e9ac3e2ab9ff89664d9b149b204ebad71c14
+  metadata.gz: 9f8caf00fc4eac2a56d69561c61e292b983626da4c25e9e96c2c25983e7e717f
+  data.tar.gz: 1ed507e15ea8a6c1fb17eb0c05d1c0f883cb19b49e8450ac3b6802e002d81104
 SHA512:
-  metadata.gz: 5c22d5b53cb2146ffd2c8bb9daee2d99410dd40ffe03d7547a89fe9ba73277094c666d9a657e0be69407dc5a2fcb6f66b2b8d7fae0d23568ee172c0d1affaa65
-  data.tar.gz: 95dc7384b03ecc1895ae908a0ced7b631d7fa124b22ae291bf02a1052547a63e18ebb6186db5449058eebee3ec9ef26ed20d163ce55891bf6dc98c1cfbb0b040
+  metadata.gz: dbdb44e7b030c7dcb6fe5438db01b44985acda2ba517001e89681471bf6f4e6199c70c47f5df8add056498f34c5a5965c86b90d812307174ca71a7382afe98a8
+  data.tar.gz: c67e0e52295a0219614ab0733d9dd35e1fcc3a835f4f3e3825c8edd2f32878c9b86e9260d835e28d9ab5ecd07ae899ff44c2dcc347e94753245c9dfa1cd5e03e

data/app/assets/stylesheets/completion_kit/application.css

@@ -1745,6 +1745,38 @@ tr:hover .ck-chip--publish {
   color: var(--ck-muted);
 }
 
+.ck-header-preview {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 0.4rem;
+  margin-top: 0.3rem;
+}
+
+.ck-header-preview__label {
+  font-family: var(--ck-mono);
+  font-size: 0.66rem;
+  letter-spacing: 0.08em;
+  text-transform: uppercase;
+  color: var(--ck-dim);
+}
+
+.ck-header-chip {
+  display: inline-flex;
+  align-items: center;
+  padding: 0.12rem 0.5rem;
+  border: 1px solid var(--ck-line-strong);
+  border-radius: 999px;
+  font-family: var(--ck-mono);
+  font-size: 0.72rem;
+  color: var(--ck-muted);
+}
+
+.ck-header-chip--special {
+  border-color: var(--ck-accent);
+  color: var(--ck-accent);
+}
+
 .ck-field--info #refresh-status,
 .ck-field--warn #refresh-status,
 .ck-field--error #refresh-status {

data/app/controllers/completion_kit/api/v1/runs_controller.rb

@@ -83,6 +83,7 @@ module CompletionKit
             judge_model: @run.judge_model,
             temperature: @run.temperature,
             output_column: @run.output_column,
+            expected_column: @run.expected_column,
             tag_names: @run.tag_names,
             status: "pending"
           )
@@ -157,7 +158,7 @@ module CompletionKit
         end
 
         def run_params
-          params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column,
+          params.permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column, :expected_column,
             metric_ids: [], tag_names: [])
         end
       end

data/app/controllers/completion_kit/runs_controller.rb

@@ -41,9 +41,16 @@ module CompletionKit
       if @run.responses.any? && run_generation_changed?
         attrs = run_params.except(:metric_ids).to_h
         attrs.delete("name") if attrs["name"].to_s == @run.name.to_s
-        new_run = Run.create!(attrs.merge(status: "pending"))
-        new_run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
-        redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
+        new_run = Run.new(attrs.merge(status: "pending"))
+        if new_run.save
+          new_run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
+          redirect_to run_path(new_run), notice: "Saved as a new run. The previous run and its results are preserved."
+        else
+          @run.assign_attributes(run_params.except(:metric_ids))
+          new_run.errors.each { |error| @run.errors.add(error.attribute, error.message) }
+          load_form_collections
+          render :edit, status: :unprocessable_entity
+        end
       elsif @run.update(run_params.except(:metric_ids))
         @run.replace_metrics!(params[:run][:metric_ids]) if params[:run].key?(:metric_ids)
         redirect_to run_path(@run), notice: "Run saved."
@@ -97,6 +104,7 @@ module CompletionKit
         judge_model: @run.judge_model,
         temperature: @run.temperature,
         output_column: @run.output_column,
+        expected_column: @run.expected_column,
         tag_names: @run.tag_names,
         status: "pending"
       )
@@ -224,13 +232,13 @@ module CompletionKit
     end
 
     def run_params
-      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column, metric_ids: [], tag_names: [])
+      params.require(:run).permit(:name, :prompt_id, :dataset_id, :judge_model, :temperature, :output_column, :expected_column, metric_ids: [], tag_names: [])
     end
 
     # Editing a run that already has results forks a new run — but only when a
     # field that affects generation or judging changed. Renaming or retagging is
     # pure metadata and updates the run in place.
-    GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature output_column].freeze
+    GENERATION_RUN_FIELDS = %i[prompt_id dataset_id judge_model temperature output_column expected_column].freeze
 
     def run_generation_changed?
       GENERATION_RUN_FIELDS.each do |field|

data/app/models/completion_kit/provider_credential.rb

@@ -65,7 +65,7 @@ module CompletionKit
     def judge_count
       model_ids = Model.where(provider: provider).pluck(:model_id)
       return 0 if model_ids.empty?
-      Run.where(judge_model: model_ids).display_scoped.distinct.count(:judge_model)
+      Run.where(judge_model: model_ids).distinct.count(:judge_model)
     end
 
     def last_used_at
@@ -74,7 +74,6 @@ module CompletionKit
       prompt_scope = Prompt.where(llm_model: model_ids).select(:id)
       Run.where("prompt_id IN (:prompts) OR judge_model IN (:models)",
                 prompts: prompt_scope, models: model_ids)
-         .display_scoped
          .where.not(status: "pending")
          .maximum(:created_at)
     end

data/app/models/completion_kit/response.rb

@@ -59,7 +59,7 @@ module CompletionKit
     private
 
     def requires_response_text?
-      succeeded? && !run&.judge_only_input_data_checks?
+      succeeded? && !run&.judge_only?
     end
 
     def broadcast_row_update

data/app/models/completion_kit/run.rb

@@ -17,9 +17,11 @@ module CompletionKit
     validates :status, inclusion: { in: STATUSES }
     validate :dataset_supplies_prompt_variables
     validate :judge_only_run_supplies_output_column
+    validate :dataset_supplies_expected_column
 
     before_validation :set_default_status, on: :create
     before_validation :set_auto_name, on: :create
+    after_create_commit :notify_host_of_creation
 
     def self.display_scoped
       filter = CompletionKit.config.runs_display_scope
@@ -198,40 +200,48 @@ module CompletionKit
         end
       end
 
-      transaction do
-        responses.destroy_all
-        update!(
-          status: "running",
-          progress_current: 0,
-          progress_total: rows.length,
-          failure_summary: nil,
-          error_message: nil
-        )
-        rows.each_with_index do |row, index|
-          input = row.empty? ? nil : row.to_json
-          attrs = {
-            status: "pending",
-            row_index: index,
-            input_data: input,
-            expected_output: row["expected_output"]
-          }
-          if judge_only?
-            attrs[:status] = "succeeded"
-            column = output_column.presence || "actual_output"
-            attrs[:response_text] = row[column].to_s if dataset && dataset.headers.include?(column)
+      failed_row = nil
+      begin
+        transaction do
+          responses.destroy_all
+          update!(
+            status: "running",
+            progress_current: 0,
+            progress_total: rows.length,
+            failure_summary: nil,
+            error_message: nil
+          )
+          rows.each_with_index do |row, index|
+            failed_row = index
+            input = row.empty? ? nil : row.to_json
+            attrs = {
+              status: "pending",
+              row_index: index,
+              input_data: input,
+              expected_output: row[expected_column.presence || "expected_output"]
+            }
+            if judge_only?
+              attrs[:status] = "succeeded"
+              column = output_column.presence || "actual_output"
+              attrs[:response_text] = row[column].to_s if dataset && dataset.headers.include?(column)
+            end
+
+            response = responses.create!(attrs)
+
+            if judge_only?
+              llm_metrics.each { |m| JudgeReviewJob.perform_later(response.id, m.id, id) } if llm_judge_configured?
+              check_metrics.each { |m| CheckReviewJob.perform_later(response.id, m.id, id) }
+            else
+              GenerateRowJob.perform_later(id, response.id)
+            end
           end
 
-          response = responses.create!(attrs)
-
-          if judge_only?
-            llm_metrics.each { |m| JudgeReviewJob.perform_later(response.id, m.id, id) } if llm_judge_configured?
-            check_metrics.each { |m| CheckReviewJob.perform_later(response.id, m.id, id) }
-          else
-            GenerateRowJob.perform_later(id, response.id)
-          end
+          RunCompletionCheckJob.perform_later(id) if judge_only?
         end
-
-        RunCompletionCheckJob.perform_later(id) if judge_only?
+      rescue ActiveRecord::RecordInvalid => e
+        reload
+        detail = e.record.errors.full_messages.to_sentence
+        return fail_with_summary!(failed_row ? "Row #{failed_row + 1}: #{detail}" : detail)
       end
 
       safely_broadcast do
@@ -324,6 +334,7 @@ module CompletionKit
         id: id, name: name, status: status, prompt_id: prompt_id,
         dataset_id: dataset_id, judge_model: judge_model, temperature: temperature,
         output_column: output_column,
+        expected_column: expected_column,
         created_at: created_at, updated_at: updated_at,
         responses_count: responses.count, avg_score: avg_score,
         check_pass_rate: check_pass_rate,
@@ -407,6 +418,12 @@ module CompletionKit
 
     private
 
+    def notify_host_of_creation
+      CompletionKit.config.on_run_created&.call(self)
+    rescue StandardError => e
+      Rails.error.report(e, handled: true, context: { hook: "on_run_created", run_id: id })
+    end
+
     def fail_with_summary!(message)
       errors.add(:base, message)
       if persisted?
@@ -457,6 +474,14 @@ module CompletionKit
       end
     end
 
+    def dataset_supplies_expected_column
+      return if expected_column.blank? || dataset.nil?
+
+      unless dataset.headers.include?(expected_column)
+        errors.add(:expected_column, "\"#{expected_column}\" is not a column on dataset \"#{dataset.name}\"")
+      end
+    end
+
     def judge_only_run_supplies_output_column
       return if prompt.present?
 

data/app/services/completion_kit/mcp_tools/datasets.rb

@@ -19,7 +19,7 @@ module CompletionKit
           handler: :get
         },
         "datasets_create" => {
-          description: "Create a dataset with CSV data",
+          description: "Create a dataset with CSV data. First row is the header. Two column names are recognized specially: \"expected_output\" is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and \"actual_output\" is a pre-made output to score in a prompt-less run. Both are overridable per run (expected_column / output_column). Every column is also available to the prompt as a variable.",
           inputSchema: {
             type: "object",
             properties: {name: {type: "string"}, csv_data: {type: "string"}, tag_names: {type: "array", items: {type: "string"}}},
@@ -42,7 +42,7 @@ module CompletionKit
           handler: :delete
         },
         "datasets_create_from_url" => {
-          description: "Create a dataset by downloading CSV from a URL instead of inlining it. Use this for large datasets: pass a public http(s) URL and the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked and the download is capped at 10MB.",
+          description: "Create a dataset by downloading CSV from a URL instead of inlining it. Use this for large datasets: pass a public http(s) URL and the server fetches the CSV directly, so the data never has to pass through the tool-call arguments. The URL is SSRF-checked and the download is capped at 10MB. First row is the header; the \"expected_output\" (answer key) and \"actual_output\" (pre-made output) columns are recognized specially, overridable per run.",
           inputSchema: {
             type: "object",
             properties: {

data/app/services/completion_kit/mcp_tools/runs.rb

@@ -22,6 +22,7 @@ module CompletionKit
               name: {type: "string"}, prompt_id: {type: "integer"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
               output_column: {type: "string", description: "Dataset column to grade when prompt_id is omitted; defaults to \"actual_output\"."},
+              expected_column: {type: "string", description: "Dataset column holding each row's answer key / ground truth, graded by checks with compare_to \"expected\" and passed to the judge; defaults to \"expected_output\"."},
               metric_ids: {type: "array", items: {type: "integer"}},
               tag_names: {type: "array", items: {type: "string"}}
             },
@@ -37,6 +38,7 @@ module CompletionKit
               id: {type: "integer"}, name: {type: "string"},
               dataset_id: {type: "integer"}, judge_model: {type: "string"},
               output_column: {type: "string"},
+              expected_column: {type: "string"},
               metric_ids: {type: "array", items: {type: "integer"}},
               tag_names: {type: "array", items: {type: "string"}}
             },
@@ -50,7 +52,7 @@ module CompletionKit
           handler: :delete
         },
         "runs_generate" => {
-          description: "Generate responses for a run using its prompt and dataset",
+          description: "Start a run. Required for every run, including score-only runs (no prompt): generates responses with the prompt when there is one, otherwise copies the graded dataset column and grades it.",
           inputSchema: {type: "object", properties: {id: {type: "integer"}}, required: ["id"]},
           handler: :generate
         }
@@ -65,7 +67,7 @@ module CompletionKit
       end
 
       def self.create(args)
-        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column"))
+        run = Run.new(args.slice("name", "prompt_id", "dataset_id", "judge_model", "output_column", "expected_column"))
         if run.save
           run.replace_metrics!(args["metric_ids"])
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
@@ -77,7 +79,7 @@ module CompletionKit
 
       def self.update(args)
         run = Run.find(args["id"])
-        if run.update(args.except("id", "metric_ids", "tag_names").slice("name", "dataset_id", "judge_model", "output_column"))
+        if run.update(args.except("id", "metric_ids", "tag_names").slice("name", "dataset_id", "judge_model", "output_column", "expected_column"))
           run.replace_metrics!(args["metric_ids"]) if args.key?("metric_ids")
           run.update!(tag_names: args["tag_names"]) if args.key?("tag_names")
           text_result(run.reload.as_json)

data/app/views/completion_kit/api_reference/_body.html.erb

@@ -126,7 +126,7 @@
       <div class="ck-api-endpoint">
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/runs</p>
         <p class="ck-meta-copy">Create a new run.</p>
-        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>)</p>
+        <p class="ck-api-params"><strong>Optional:</strong>&ensp;<code>name</code>, <code>prompt_id</code>, <code>dataset_id</code>, <code>metric_ids</code>, <code>judge_model</code>, <code>output_column</code> (score existing outputs: omit <code>prompt_id</code> and grade a dataset column instead, default <code>actual_output</code>), <code>expected_column</code> (dataset column holding each row's answer key, given to the judge and to checks that compare against the row's expected value, default <code>expected_output</code>)</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/runs \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"prompt_id\": 1, \"dataset_id\": 1, \"metric_ids\": [1, 2]}'" %>
       </div>
       <div class="ck-api-endpoint">
@@ -203,6 +203,7 @@
         <p class="ck-api-method"><span class="ck-chip ck-chip--soft">POST</span> /api/v1/datasets</p>
         <p class="ck-meta-copy">Create a dataset from inline CSV or an uploaded CSV file.</p>
         <p class="ck-api-params"><strong>Required:</strong>&ensp;<code>name</code>, and either <code>csv_data</code> (inline CSV) or a multipart <code>file</code> (CSV upload, preferred for large datasets)</p>
+        <p class="ck-meta-copy">First row is the header. Two column names are recognized specially: <code>expected_output</code> is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and <code>actual_output</code> is a pre-made output to score in a prompt-less run. Both are overridable per run via <code>expected_column</code> and <code>output_column</code>.</p>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/datasets \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -H \"Content-Type: application/json\" \\\n  -d '{\"name\": \"tickets\", \"csv_data\": \"text,expected_output\\\\nHello,Hi\"}'" %>
         <%= render "completion_kit/api_reference/example", base_url: base_url, token: token, real_token: real_token, cmd: "curl -X POST #{base_url}/api/v1/datasets \\\n  -H \"Authorization: Bearer #{token}\" \\\n  -F \"name=tickets\" \\\n  -F \"file=@tickets.csv\"" %>
       </div>

data/app/views/completion_kit/datasets/_form.html.erb

@@ -19,8 +19,13 @@
 
     <div class="ck-field">
       <%= form.label :csv_data, "CSV data", class: "ck-label" %>
-      <%= form.text_area :csv_data, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "content,audience\nFirst ticket text,internal\nSecond ticket text,customer", **ck_field_aria(form, :csv_data) %>
+      <%= form.text_area :csv_data, rows: 12, class: "ck-input ck-input--area ck-input--code", placeholder: "content,audience\nFirst ticket text,internal\nSecond ticket text,customer", id: "dataset_csv_data", **ck_field_aria(form, :csv_data) %>
       <%= ck_field_error(form, :csv_data) %>
+      <p class="ck-field-hint">First row is the header. Two column names are recognized specially: <code>expected_output</code> is each row's answer key (ground truth) given to the judge and to checks that compare against the row's expected value, and <code>actual_output</code> is a pre-made output to score when you run without a prompt. Both are overridable per run. Every column is also available to the prompt as a variable.</p>
+      <div class="ck-header-preview" id="dataset-header-preview" hidden>
+        <span class="ck-header-preview__label">Detected columns</span>
+        <span id="dataset-header-chips"></span>
+      </div>
     </div>
 
     <%= render "completion_kit/tags/picker", record: dataset, param_namespace: :dataset %>
@@ -47,4 +52,32 @@
       <%= form.submit(dataset.persisted? ? "Save dataset" : "Create dataset", class: ck_button_classes(:dark)) %>
     </div>
   </div>
+
+  <script>
+function ckDatasetHeaderPreview() {
+  var area = document.getElementById('dataset_csv_data');
+  var box = document.getElementById('dataset-header-preview');
+  var chips = document.getElementById('dataset-header-chips');
+  if (!area || !box || !chips) return;
+  var recognized = { expected_output: 'answer key', actual_output: 'pre-made output' };
+  var firstLine = (area.value.split(/\r?\n/).find(function(l) { return l.trim() !== ''; }) || '');
+  var headers = firstLine.split(',').map(function(h) { return h.trim().replace(/^"|"$/g, ''); }).filter(Boolean);
+  if (headers.length === 0) { box.hidden = true; chips.textContent = ''; return; }
+  box.hidden = false;
+  chips.textContent = '';
+  headers.forEach(function(h) {
+    var role = recognized[h];
+    var chip = document.createElement('span');
+    chip.className = 'ck-header-chip' + (role ? ' ck-header-chip--special' : '');
+    chip.textContent = role ? h + ' · ' + role : h;
+    chips.appendChild(chip);
+  });
+}
+document.addEventListener('turbo:load', ckDatasetHeaderPreview);
+(function() {
+  var area = document.getElementById('dataset_csv_data');
+  if (area) area.addEventListener('input', ckDatasetHeaderPreview);
+})();
+ckDatasetHeaderPreview();
+  </script>
 <% end %>

data/app/views/completion_kit/runs/_form.html.erb

@@ -75,6 +75,14 @@
       <p class="ck-field-hint" id="dataset-hint"></p>
     </div>
 
+    <div class="ck-field" id="expected-column-field">
+      <%= form.label :expected_column, "Answer-key column", class: "ck-label" %>
+      <%= form.text_field :expected_column, class: "ck-input", id: "run_expected_column", placeholder: "expected_output", **ck_field_aria(form, :expected_column) %>
+      <%= ck_field_error(form, :expected_column) %>
+      <p class="ck-field-hint" id="expected-column-hint">Optional. The dataset column holding each row's correct answer, used by the judge and by checks set to compare against the row's expected value. Defaults to <code>expected_output</code>.</p>
+      <p class="ck-field-hint" id="expected-column-warn"></p>
+    </div>
+
     <div class="ck-field">
       <label class="ck-label" for="run_temperature" style="position: relative;">
         Temperature<span class="ck-info-toggle" tabindex="0">?</span><span class="ck-info-popup">Controls how random the model's output is. Lower values are more focused and deterministic — the model picks the most likely words. Higher values are more varied and creative, with more risk of odd phrasing. Most LLMs default to 1.0; for evaluation, try a few values and see how your prompt holds up. Newer reasoning models (Claude Opus 4.7, GPT-5 family, etc.) ignore temperature entirely — CompletionKit detects this and re-sends without the parameter.</span>
@@ -152,7 +160,7 @@
 
         <div class="ck-metric-checkboxes" data-metric-checkboxes>
           <% @all_metrics.each do |metric| %>
-            <label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>">
+            <label class="ck-checkbox-label" data-metric-tags="<%= metric.tag_names.join(",") %>" data-compare-expected="<%= metric.check? && metric.check_config.to_h["compare_to"] == "expected" ? "1" : "0" %>">
               <%= check_box_tag "run[metric_ids][]", metric.id, run.metric_ids.include?(metric.id), class: "ck-checkbox", id: "run_metric_#{metric.id}" %>
               <span class="ck-checkbox-label__box" aria-hidden="true"></span>
               <span class="ck-checkbox-label__body">
@@ -235,6 +243,26 @@ function updateRunForm() {
     if (datasetHint) datasetHint.textContent = 'This prompt uses variables. Select a dataset to provide values.';
   }
 
+  var expectedWarn = document.getElementById('expected-column-warn');
+  var expectedField = document.getElementById('expected-column-field');
+  var expectedColEl = document.getElementById('run_expected_column');
+  if (expectedField) expectedField.className = 'ck-field';
+  if (expectedWarn) expectedWarn.textContent = '';
+  var wantsExpected = false;
+  document.querySelectorAll('input[name="run[metric_ids][]"]:checked').forEach(function(cb) {
+    var lbl = cb.closest('[data-compare-expected]');
+    if (lbl && lbl.dataset.compareExpected === '1') wantsExpected = true;
+  });
+  if (wantsExpected && dataset && datasetEl) {
+    var expOption = datasetEl.options[datasetEl.selectedIndex];
+    var expHeaders = (expOption && expOption.dataset.headers ? expOption.dataset.headers.split(/,\s*/) : []).filter(Boolean);
+    var expCol = ((expectedColEl && expectedColEl.value) || 'expected_output').trim();
+    if (expHeaders.indexOf(expCol) === -1) {
+      if (expectedField) expectedField.className = 'ck-field ck-field--error';
+      if (expectedWarn) expectedWarn.textContent = "A selected check grades against each row's expected value, but this dataset has no \"" + expCol + "\" column, so every row would fail. Add the column or point this at the right one.";
+    }
+  }
+
   var summary = document.getElementById('prompt-summary');
   if (summary) {
     if (selectedOption && selectedOption.value) {

data/db/migrate/20260706000001_add_expected_column_to_completion_kit_runs.rb

@@ -0,0 +1,5 @@
+class AddExpectedColumnToCompletionKitRuns < ActiveRecord::Migration[8.1]
+  def change
+    add_column :completion_kit_runs, :expected_column, :string
+  end
+end

data/lib/completion_kit/version.rb

@@ -1,3 +1,3 @@
 module CompletionKit
-  VERSION = "0.23.0"
+  VERSION = "0.24.0"
 end

data/lib/completion_kit.rb

@@ -15,6 +15,7 @@ module CompletionKit
     attr_accessor :allow_loopback_endpoints
     attr_accessor :judge_agreement_enabled
     attr_accessor :judge_examples_from_reviews
+    attr_accessor :on_run_created
 
     def initialize
       @openai_api_key = ENV['OPENAI_API_KEY']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: completion-kit
 version: !ruby/object:Gem::Version
-  version: 0.23.0
+  version: 0.24.0
 platform: ruby
 authors:
 - Damien Bastin
@@ -161,14 +161,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '8.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '8.0'
 - !ruby/object:Gem::Dependency
   name: factory_bot_rails
   requirement: !ruby/object:Gem::Requirement
@@ -475,6 +475,7 @@ files:
 - db/migrate/20260629000001_add_check_type_to_completion_kit_metrics.rb
 - db/migrate/20260629000002_add_check_type_to_completion_kit_metric_versions.rb
 - db/migrate/20260629000003_add_passed_to_completion_kit_reviews.rb
+- db/migrate/20260706000001_add_expected_column_to_completion_kit_runs.rb
 - lib/completion-kit.rb
 - lib/completion_kit.rb
 - lib/completion_kit/concurrency_check.rb