ruby_llm-evals 0.0.1 → 0.1.0

data/app/views/ruby_llm/evals/prompts/compare.html.erb

@@ -0,0 +1,90 @@
+<% content_for :title, "#{@prompt.name} - Runs Comparison" %>
+
+<h1 class="title"><%= @prompt.name %> - Runs Comparison</h1>
+
+<% if @runs.any? %>
+  <div style="overflow-x: auto;">
+    <table class="table is-bordered is-striped is-fullwidth">
+      <thead>
+        <tr>
+          <th>Metric</th>
+          <% @runs.each do |run| %>
+            <th>
+              <%= link_to "Run ##{run.id}", run_path(run) %>
+              <div class="is-size-7 has-text-grey">
+                <%= run.created_at %>
+              </div>
+            </th>
+          <% end %>
+        </tr>
+      </thead>
+      <tbody>
+        <tr>
+          <th>Provider/Model</th>
+          <% @runs.each do |run| %>
+            <td><%= run.provider %>/<%= run.model %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Temperature</th>
+          <% @runs.each do |run| %>
+            <td><%= run.temperature || "N/A" %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Instructions</th>
+          <% @runs.each do |run| %>
+            <td><%= truncate run.instructions %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Message</th>
+          <% @runs.each do |run| %>
+            <td><%= truncate run.message %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Accuracy</th>
+          <% @runs.each do |run| %>
+            <td><%= accuracy(run) %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Cost</th>
+          <% @runs.each do |run| %>
+            <td>$<%= run.total_cost %></td>
+          <% end %>
+        </tr>
+
+        <tr>
+          <th>Duration</th>
+          <% @runs.each do |run| %>
+            <td><%= duration(run) %></td>
+          <% end %>
+        </tr>
+
+        <% if @samples.any? %>
+          <% @samples.each_with_index do |sample, sample_index| %>
+            <tr>
+              <th>
+                Sample #<%= sample.id %>
+              </th>
+              <% @runs.each do |run| %>
+                <td class="has-text-centered">
+                  <%= status_indicator run.prompt_executions.find { |pe| pe.sample.id == sample.id } %>
+                </td>
+              <% end %>
+            </tr>
+          <% end %>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+<% else %>
+  <div class="mt-6 has-text-centered is-size-4 has-text-grey">There are no runs</div>
+<% end %>

data/app/views/ruby_llm/evals/prompts/edit.html.erb

@@ -0,0 +1,5 @@
+<% content_for :title, "Editing prompt" %>
+
+<h1 class="title">Editing prompt</h1>
+
+<%= render "form", prompt: @prompt %>

data/app/views/ruby_llm/evals/prompts/index.html.erb

@@ -0,0 +1,32 @@
+<% content_for :title, "Prompts" %>
+
+<div class="level">
+  <div class="level-left">
+    <%= render "filters", clear_path: prompts_path %>
+  </div>
+  <div class="level-right is-align-self-flex-end">
+    <%= link_to "New prompt", new_prompt_path, class: "button is-link" %>
+  </div>
+</div>
+
+<% if @prompts.records.any? %>
+  <table class="table is-hoverable is-fullwidth">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Provider/Model</th>
+        <th>Instructions</th>
+        <th>Message</th>
+        <th>Runs</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody>
+      <%= render partial: "ruby_llm/evals/prompts/prompt", collection: @prompts.records %>
+    </tbody>
+  </table>
+<% else %>
+  <div class="mt-6 has-text-centered is-size-4 has-text-grey">There are no prompts</div>
+<% end %>
+
+<%= render "ruby_llm/evals/application/pagination", page: @prompts %>

data/app/views/ruby_llm/evals/prompts/new.html.erb

@@ -0,0 +1,5 @@
+<% content_for :title, "New prompt" %>
+
+<h1 class="title">New prompt</h1>
+
+<%= render "form", prompt: @prompt %>

data/app/views/ruby_llm/evals/prompts/show.html.erb

@@ -0,0 +1,107 @@
+<% content_for :title, @prompt.name %>
+
+<h1 class="title"><%= @prompt.name %></h1>
+
+<table class="table">
+  <tbody>
+    <tr>
+      <th>Slug</th>
+      <td><%= @prompt.slug %></td>
+    </tr>
+    <tr>
+      <th>Provider</th>
+      <td><%= @prompt.provider %></td>
+    </tr>
+    <tr>
+      <th>Model</th>
+      <td><%= @prompt.model %></td>
+    </tr>
+    <tr>
+      <th>Temperature</th>
+      <td><%= @prompt.temperature %></td>
+    </tr>
+    <tr>
+      <th>Params</th>
+      <td><%= json @prompt.params %></td>
+    </tr>
+    <tr>
+      <th>Tools</th>
+      <td><%= @prompt.tools.try :to_sentence %></td>
+    </tr>
+    <tr>
+      <th>Schema</th>
+      <td>
+        <% if @prompt.schema_other.present? %>
+          <%= json @prompt.schema_other %>
+        <% elsif @prompt.schema.present? %>
+          <%= @prompt.schema %>
+        <% end %>
+      </td>
+    </tr>
+    <tr>
+      <th>Instructions</th>
+      <td style="white-space: pre-wrap;"><%= @prompt.instructions %></td>
+    </tr>
+    <tr>
+      <th>Message</th>
+      <td style="white-space: pre-wrap;"><%= @prompt.message %></td>
+    </tr>
+  </tbody>
+</table>
+
+<h2 class="mt-6 subtitle">Samples</h2>
+
+<% if @prompt.samples.any? %>
+  <table class="table is-fullwidth">
+    <thead>
+      <tr>
+        <th>Sample</th>
+        <th>Variables</th>
+        <th>Eval type</th>
+        <th>Expected output</th>
+        <th>Files</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @prompt.samples.each do |sample| %>
+        <tr>
+          <td>#<%= sample.id %></td>
+          <td><%= json sample.variables %></td>
+          <td>
+            <%= sample.eval_type %>
+            <% if sample.llm_judge? %>
+              <div>(<%= sample.judge_provider %>/<%= sample.judge_model %>)</div>
+            <% end %>
+          </td>
+          <td><%= expected_output sample %></td>
+          <td>
+            <ul>
+              <% sample.files.each do |file| %>
+                <li>
+                  <% if file.previewable? %>
+                    <%= image_tag main_app.url_for(file.preview(resize_to_limit: [400, 400]).processed) %>
+                  <% elsif file.variable? %>
+                    <%= image_tag main_app.url_for(file.variant(resize_to_limit: [400, 400]).processed) %>
+                  <% else %>
+                    <%= link_to file.filename, main_app.url_for(file) %>
+                  <% end %>
+                  <div><%= link_to file.filename, main_app.rails_blob_path(file, disposition: "attachment") %> (<%= number_to_human_size(file.byte_size) %>)</div>
+                </li>
+              <% end %>
+            </ul>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% else %>
+  <div class="mt-6 has-text-centered is-size-4 has-text-grey">There are no samples</div>
+<% end %>
+
+<div class="mt-6 buttons">
+  <% if @prompt.runs.any? %>
+    <%= link_to "Compare runs", compare_prompt_path(@prompt), class: "button" %>
+  <% end %>
+  <%= link_to "Edit this prompt", edit_prompt_path(@prompt), class: "button" %>
+  <%= button_to "Destroy this prompt", @prompt, method: :delete, class: "button is-danger" %>
+</div>

data/app/views/ruby_llm/evals/runs/_filters.html.erb

@@ -0,0 +1,17 @@
+<div class="level">
+  <div class="level-left">
+    <%= form_with scope: :filter, url: runs_path, method: :get, data: {controller: "auto-submit"} do |form| %>
+      <div class="field is-grouped">
+        <div>
+          <%= form.label :ruby_llm_evals_prompt_id, "Prompt", class: "label" %>
+          <div class="select">
+            <%= form.collection_select :ruby_llm_evals_prompt_id, RubyLLM::Evals::Prompt.all, :id, :name, {selected: filter_param.dig(:filter, :ruby_llm_evals_prompt_id), include_blank: true}, data: {action: "auto-submit#submit"} %>
+          </div>
+        </div>
+        <div class="is-align-self-flex-end">
+          <%= link_to "Clear", clear_path, class: "button" %>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>

data/app/views/ruby_llm/evals/runs/_run.html.erb

@@ -0,0 +1,13 @@
+<tr>
+  <td><%= link_to "##{run.id}", run_path(run) %></td>
+  <td><%= link_to run.prompt.name, run.prompt %></td>
+  <td><%= run.provider %>/<%= run.model %></td>
+  <td><%= truncate run.instructions %></td>
+  <td><%= truncate run.message %></td>
+  <td><%= run.started_at %></td>
+  <td><%= run.ended_at %></td>
+  <td><%= duration run %></td>
+  <td><%= accuracy run %></td>
+  <td>$<%= run.cost %></td>
+  <td><%= button_to "Destroy", run, method: :delete, class: "button is-danger" %></td>
+</tr>

data/app/views/ruby_llm/evals/runs/index.html.erb

@@ -0,0 +1,30 @@
+<% content_for :title, "Runs" %>
+
+<%= render "filters", clear_path: runs_path %>
+
+<% if @runs.records.any? %>
+  <table class="table is-hoverable is-fullwidth">
+    <thead>
+      <tr>
+        <th>Run</th>
+        <th>Prompt</th>
+        <th>Provider/Model</th>
+        <th>Instructions</th>
+        <th>Message</th>
+        <th>Started at</th>
+        <th>Ended at</th>
+        <th>Duration</th>
+        <th>Accuracy</th>
+        <th>Cost</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody>
+      <%= render partial: "ruby_llm/evals/runs/run", collection: @runs.records %>
+    </tbody>
+  </table>
+<% else %>
+  <div class="mt-6 has-text-centered is-size-4 has-text-grey">There are no runs</div>
+<% end %>
+
+<%= render "ruby_llm/evals/application/pagination", page: @runs %>

data/app/views/ruby_llm/evals/runs/show.html.erb

@@ -0,0 +1,188 @@
+<% content_for :title, "Run ##{@run.id}" %>
+
+<h1 class="title">Run #<%= @run.id %></h1>
+
+<table class="table">
+  <tbody>
+    <tr>
+      <th>Prompt</th>
+      <td><%= link_to @run.prompt.name, @run.prompt %></td>
+    </tr>
+    <tr>
+      <th>Provider/Model</th>
+      <td><%= @run.provider %>/<%= @run.model %></td>
+    </tr>
+    <tr>
+      <th>Temperature</th>
+      <td><%= @run.temperature %></td>
+    </tr>
+    <tr>
+      <th>Params</th>
+      <td><%= json @run.params %></td>
+    </tr>
+    <tr>
+      <th>Tools</th>
+      <td><%= @run.tools.try :to_sentence %></td>
+    </tr>
+    <tr>
+      <th>Schema</th>
+      <td>
+        <% if @run.schema_other.present? %>
+          <%= json @run.schema_other %>
+        <% elsif @run.schema.present? %>
+          <%= @run.schema %>
+        <% end %>
+      </td>
+    </tr>
+    <tr>
+      <th>Instructions</th>
+      <td style="white-space: pre-wrap;"><%= @run.instructions %></td>
+    </tr>
+    <tr>
+      <th>Message</th>
+      <td style="white-space: pre-wrap;"><%= @run.message %></td>
+    </tr>
+    <tr>
+      <th>ActiveJob ID</th>
+      <td><%= @run.active_job_id %></td>
+    </tr>
+    <tr>
+      <th>Started at</th>
+      <td><%= @run.started_at %></td>
+    </tr>
+    <tr>
+      <th>Ended at</th>
+      <td><%= @run.ended_at %></td>
+    </tr>
+    <tr>
+      <th>Duration</th>
+      <td><%= duration @run %></td>
+    </tr>
+    <tr>
+      <th>Accuracy</th>
+      <td><%= accuracy @run %></td>
+    </tr>
+    <tr>
+      <th>Cost</th>
+      <td>$<%= @run.cost %></td>
+    </tr>
+    <% if @run.judge_cost > 0 %>
+      <tr>
+        <th>Judge Cost</th>
+        <td>$<%= @run.judge_cost %></td>
+      </tr>
+      <tr>
+        <th>Total Cost</th>
+        <td>$<%= @run.total_cost %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<h2 class="mt-6 subtitle">Prompt executions</h2>
+
+<table class="table is-fullwidth">
+  <thead>
+    <tr>
+      <th>Active Job ID</th>
+      <th>Variables</th>
+      <th>With</th>
+      <th>Output message</th>
+      <th>Eval type</th>
+      <th>Expected output</th>
+      <th>Tokens (input/output)</th>
+      <th>Duration</th>
+      <th>Cost</th>
+      <th>Passed?</th>
+      <th></th>
+    </tr>
+  </thead>
+  <tbody>
+  <% @run.prompt_executions.each do |prompt_execution| %>
+      <tr>
+        <td>
+          <%= prompt_execution.active_job_id %>
+          <% if prompt_execution.error_message.present? %>
+            <span title="<%= prompt_execution.error_message %>">⚠️</span>
+          <% end %>
+        </td>
+        <td><%= json prompt_execution.variables %></td>
+        <td>
+          <ul>
+            <% prompt_execution.files.each do |file| %>
+              <li>
+                <% if file.previewable? %>
+                  <%= image_tag main_app.url_for(file.preview(resize_to_limit: [400, 400]).processed) %>
+                <% elsif file.variable? %>
+                  <%= image_tag main_app.url_for(file.variant(resize_to_limit: [400, 400]).processed) %>
+                <% else %>
+                  <%= link_to file.filename, main_app.url_for(file) %>
+                <% end %>
+                <div><%= link_to file.filename, main_app.rails_blob_path(file, disposition: "attachment") %> (<%= number_to_human_size(file.byte_size) %>)</div>
+              </li>
+            <% end %>
+          </ul>
+        </td>
+        <td>
+          <% if @run.schema_other.present? || @run.schema.present? %>
+            <% if prompt_execution.message.present? %>
+              <%= json JSON.parse(prompt_execution.message) %>
+            <% end %>
+          <% else %>
+            <div style="white-space: pre-wrap;"><%= prompt_execution.message %></div>
+          <% end %>
+        </td>
+        <td>
+          <%= prompt_execution.eval_type %>
+          <% if prompt_execution.llm_judge? %>
+            <div>(<%= prompt_execution.judge_provider %>/<%= prompt_execution.judge_model %>)</div>
+          <% end %>
+        </td>
+        <td>
+          <% if prompt_execution.llm_judge? %>
+            <div><strong>Criteria:</strong> <%= prompt_execution.expected_output %></div>
+            <% if prompt_execution.judge_message&.[]("explanation").present? %>
+              <div><strong>Explanation:</strong> <%= prompt_execution.judge_message["explanation"] %></div>
+            <% end %>
+          <% else %>
+            <%= expected_output prompt_execution %>
+          <% end %>
+        </td>
+        <td>
+          <div><%= prompt_execution.input %>/<%= prompt_execution.output %></div>
+          <% if prompt_execution.llm_judge? %>
+            <div><%= prompt_execution.judge_input %>/<%= prompt_execution.judge_output %> (judge)</div>
+          <% end %>
+        </td>
+        <td><%= duration prompt_execution %></td>
+        <td>
+          <div>$<%= prompt_execution.cost %></div>
+          <% if prompt_execution.llm_judge? %>
+            <div>$<%= prompt_execution.judge_cost %> (judge)</div>
+          <% end %>
+        </td>
+        <td><%= status_indicator(prompt_execution) %></td>
+        <td>
+          <div class="buttons">
+            <% if prompt_execution.error_message.present? %>
+              <%= button_to "Retry", retry_prompt_execution_path(prompt_execution), method: :post, class: "button is-warning is-light" %>
+            <% end %>
+            <% if prompt_execution.sample.human_judge? %>
+              <% if prompt_execution.passed.nil? %>
+                <%= button_to "Pass", prompt_execution_passage_path(prompt_execution), method: :post, class: "button is-success is-light" %>
+                <%= button_to "Fail", prompt_execution_failure_path(prompt_execution), method: :post, class: "button is-danger is-light" %>
+              <% else %>
+                <% target_path = prompt_execution.passed? ? prompt_execution_failure_path(prompt_execution) : prompt_execution_passage_path(prompt_execution) %>
+                <%= button_to "Toggle", target_path, method: :post, class: "button" %>
+              <% end %>
+            <% end %>
+          </div>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+
+<div class="mt-6 buttons">
+  <%= button_to "Destroy this run", @run, method: :delete, class: "button is-danger" %>
+</div>

data/app/views/ruby_llm/evals/samples/_form.html.erb

@@ -0,0 +1,88 @@
+<div class="box nested-form-wrapper" data-new-record="<%= form.object.new_record? %>" data-controller="eval-type-selector">
+  <div class="columns">
+    <div class="column field">
+      <%= form.label :variables, class: "label" %>
+      <div data-controller="json-editor" data-json-editor-height-value="150px">
+        <div data-json-editor-target="editor" class="json-editor"></div>
+        <%= form.text_area :variables, value: form.object.variables&.to_json, class: "textarea", data: { json_editor_target: "textarea" } %>
+        <p class="help is-danger is-hidden" data-json-editor-target="error"></p>
+      </div>
+      <p class="help is-danger"><%= form.object.errors.full_messages_for(:variables).join(", ") %></p>
+    </div>
+
+    <div class="column">
+      <div class="field">
+        <%= form.label :eval_type, class: "label" %>
+        <div class="select is-fullwidth">
+          <%= form.select :eval_type, RubyLLM::Evals::Sample.eval_types, {}, data: { action: "change->eval-type-selector#toggle", eval_type_selector_target: "select" } %>
+        </div>
+        <p class="help is-danger"><%= form.object.errors.full_messages_for(:eval_type).join(", ") %></p>
+      </div>
+
+      <div data-eval-type-selector-target="judgeFields" data-controller="provider-model" data-provider-model-models-by-provider-value="<%= models_by_provider_data %>" style="<%= 'display: none;' unless form.object.llm_judge? %>">
+        <div class="field">
+          <%= form.label :judge_provider, class: "label" %>
+          <div class="select is-fullwidth">
+            <%= form.select :judge_provider, options_for_select(RubyLLM.providers.map { [_1.name, _1.slug] }, form.object.judge_provider), { include_blank: true }, data: { provider_model_target: "provider", action: "change->provider-model#provider-changed" } %>
+          </div>
+          <p class="help is-danger"><%= form.object.errors.full_messages_for(:judge_provider).join(", ") %></p>
+        </div>
+
+        <div class="field">
+          <%= form.label :judge_model, class: "label" %>
+          <div class="select is-fullwidth is-hidden" data-provider-model-target="modelSelectWrapper">
+            <%= form.select :judge_model, [], {}, { class: "input", name: nil, data: { provider_model_target: "modelSelect", action: "change->provider-model#syncToInput" } } %>
+          </div>
+          <%= form.text_field :judge_model, class: "input", data: { provider_model_target: "modelInput" } %>
+          <p class="help is-danger"><%= form.object.errors.full_messages_for(:judge_model).join(", ") %></p>
+        </div>
+      </div>
+    </div>
+
+    <div class="column field" data-eval-type-selector-target="expectedOutput" style="<%= 'display: none;' if form.object.human_judge? %>">
+      <%= form.label :expected_output, class: "label" %>
+      <%= form.text_area :expected_output, class: "textarea" %>
+      <p class="help is-danger"><%= form.object.errors.full_messages_for(:expected_output).join(", ") %></p>
+    </div>
+
+    <div class="column field">
+      <%= form.label :files, "Attach files", class: "label" %>
+
+      <% if form.object.persisted? && form.object.files.attached? %>
+        <% form.object.files.each do |file| %>
+          <%= form.hidden_field :files, multiple: true, value: file.signed_id %>
+        <% end %>
+      <% end %>
+
+      <div class="file has-name is-fullwidth" data-controller="file-input">
+        <label class="file-label">
+          <%= form.file_field :files, multiple: true, class: "file-input", data: { action: "change->file-input#update", file_input_target: "input" } %>
+          <span class="file-cta">
+            <span class="file-label">Choose files...</span>
+          </span>
+          <span class="file-name" data-file-input-target="name">No files selected</span>
+        </label>
+      </div>
+
+      <% if form.object.persisted? && form.object.files.any? %>
+        <div>
+          <p>Current attachments:</p>
+          <ul>
+            <% form.object.files.each do |file| %>
+              <li>
+                <span><%= file.filename %> (<%= number_to_human_size(file.byte_size) %>)</span>
+                <%= link_to "Remove", main_app.rails_blob_path(file, disposition: "attachment"), data: { turbo_method: :delete, turbo_confirm: "Are you sure?" } %>
+              </li>
+            <% end %>
+          </ul>
+        </div>
+      <% end %>
+    </div>
+  </div>
+
+  <div class="field">
+    <button type="button" class="button is-danger is-outlined" data-action="nested-form#remove">Remove sample</button>
+  </div>
+
+  <%= form.hidden_field :_destroy %>
+</div>

data/config/importmap.rb

@@ -0,0 +1,13 @@
+pin "@hotwired/turbo-rails", to: "turbo.min.js", preload: true
+pin "@hotwired/stimulus", to: "stimulus.min.js", preload: true
+pin "@hotwired/stimulus-loading", to: "stimulus-loading.js", preload: true
+pin "@stimulus-components/auto-submit", to: "https://ga.jspm.io/npm:@stimulus-components/auto-submit@6.0.0/dist/stimulus-auto-submit.mjs"
+pin "@stimulus-components/rails-nested-form", to: "https://ga.jspm.io/npm:@stimulus-components/rails-nested-form@5.0.0/dist/stimulus-rails-nested-form.mjs", preload: true
+
+pin "ace-builds", to: "https://cdn.jsdelivr.net/npm/ace-builds@1.35.5/src-min-noconflict/ace.min.js", preload: true
+pin "ace-mode-json", to: "https://cdn.jsdelivr.net/npm/ace-builds@1.35.5/src-min-noconflict/mode-json.min.js", preload: true
+pin "ace-theme-github", to: "https://cdn.jsdelivr.net/npm/ace-builds@1.35.5/src-min-noconflict/theme-github.min.js", preload: true
+pin "ace-theme-github-dark", to: "https://cdn.jsdelivr.net/npm/ace-builds@1.35.5/src-min-noconflict/theme-github_dark.min.js", preload: true
+
+pin "application", to: "ruby_llm/evals/application.js", preload: true
+pin_all_from RubyLLM::Evals::Engine.root.join("app/javascript/ruby_llm/evals/controllers"), under: "controllers", to: "ruby_llm/evals/controllers"

data/config/locales/en.yml

@@ -0,0 +1,7 @@
+en:
+  activemodel:
+    errors:
+      messages:
+        invalid_json: "is invalid JSON"
+        must_be_json_object: "must be a JSON object"
+        must_be_json_array: "must be a JSON array"

data/config/routes.rb

@@ -1,2 +1,21 @@
-Rails.application.routes.draw do
+RubyLLM::Evals::Engine.routes.draw do
+  resources :runs, only: %i[destroy index show]
+
+  resources :prompt_executions, only: [] do
+    scope module: :prompt_executions do
+      resource :failure, only: %i[create]
+      resource :passage, only: %i[create]
+      resource :retry, only: %i[create]
+    end
+  end
+
+  resources :prompts do
+    resources :runs, only: %i[create]
+
+    member do
+      get :compare
+    end
+  end
+
+  root to: "prompts#index"
 end

data/db/migrate/20251022211228_create_ruby_llm_evals_prompts.rb

@@ -0,0 +1,21 @@
+class CreateRubyLLMEvalsPrompts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ruby_llm_evals_prompts do |t|
+      t.string :name, null: false
+      t.string :slug, null: false
+      t.string :provider, null: false
+      t.string :model, null: false
+      t.float :temperature
+      t.json :params
+      t.json :tools
+      t.string :schema
+      t.json :schema_other
+      t.text :instructions
+      t.text :message
+
+      t.timestamps
+    end
+    add_index :ruby_llm_evals_prompts, :name, unique: true
+    add_index :ruby_llm_evals_prompts, :slug, unique: true
+  end
+end

data/db/migrate/20251022211229_create_ruby_llm_evals_samples.rb

@@ -0,0 +1,14 @@
+class CreateRubyLLMEvalsSamples < ActiveRecord::Migration[7.0]
+  def change
+    create_table :ruby_llm_evals_samples do |t|
+      t.references :ruby_llm_evals_prompt, null: false, foreign_key: true
+      t.string :eval_type, null: false
+      t.text :expected_output
+      t.string :judge_model
+      t.string :judge_provider
+      t.json :variables
+
+      t.timestamps
+    end
+  end
+end