prompt_engine 1.0.0

data/app/views/prompt_engine/eval_sets/compare.html.erb

@@ -0,0 +1,229 @@
+<div class="admin-header">
+  <div>
+    <h1>Compare Evaluation Runs</h1>
+    <p class="text-muted">
+      Comparing runs from <%= @run1.created_at.strftime("%b %d, %Y %I:%M %p") %> 
+      and <%= @run2.created_at.strftime("%b %d, %Y %I:%M %p") %>
+    </p>
+  </div>
+  <div class="btn-group">
+    <%= link_to "Back to Eval Set", prompt_eval_set_path(@prompt, @eval_set), 
+        class: "btn btn--secondary btn--medium" %>
+  </div>
+</div>
+
+<!-- Overall Comparison Summary -->
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Performance Comparison</h3>
+  </div>
+  <div class="card__body">
+    <div class="comparison-grid">
+      <div class="comparison-column">
+        <div class="comparison-header">
+          <h4>Run 1</h4>
+          <span class="table__badge table__badge--info">v<%= @run1.prompt_version.version_number %></span>
+          <span class="text-muted"><%= @run1.created_at.strftime("%b %d, %Y %I:%M %p") %></span>
+        </div>
+        <div class="comparison-metrics">
+          <div class="metric">
+            <div class="metric__label">Success Rate</div>
+            <div class="metric__value metric__value--large">
+              <%= number_to_percentage(@run1_success_rate, precision: 1) %>
+            </div>
+          </div>
+          <div class="metric">
+            <div class="metric__label">Passed Tests</div>
+            <div class="metric__value">
+              <%= @run1.passed_count %> / <%= @run1.total_count %>
+            </div>
+          </div>
+          <div class="metric">
+            <div class="metric__label">Run Time</div>
+            <div class="metric__value">
+              <% if @run1.completed_at && @run1.started_at %>
+                <%= distance_of_time_in_words(@run1.started_at, @run1.completed_at) %>
+              <% else %>
+                —
+              <% end %>
+            </div>
+          </div>
+        </div>
+      </div>
+      
+      <div class="comparison-divider">
+        <div class="comparison-indicator">
+          <% if @success_rate_diff > 0 %>
+            <span class="indicator indicator--success">
+              <svg class="indicator__icon" viewBox="0 0 20 20" fill="currentColor">
+                <path fill-rule="evenodd" d="M5.293 9.707a1 1 0 010-1.414l4-4a1 1 0 011.414 0l4 4a1 1 0 01-1.414 1.414L11 7.414V15a1 1 0 11-2 0V7.414L6.707 9.707a1 1 0 01-1.414 0z" clip-rule="evenodd" />
+              </svg>
+              +<%= number_to_percentage(@success_rate_diff.abs, precision: 1) %>
+            </span>
+          <% elsif @success_rate_diff < 0 %>
+            <span class="indicator indicator--danger">
+              <svg class="indicator__icon" viewBox="0 0 20 20" fill="currentColor">
+                <path fill-rule="evenodd" d="M14.707 10.293a1 1 0 010 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 111.414-1.414L9 12.586V5a1 1 0 012 0v7.586l2.293-2.293a1 1 0 011.414 0z" clip-rule="evenodd" />
+              </svg>
+              <%= number_to_percentage(@success_rate_diff.abs, precision: 1) %>
+            </span>
+          <% else %>
+            <span class="indicator indicator--neutral">
+              <svg class="indicator__icon" viewBox="0 0 20 20" fill="currentColor">
+                <path fill-rule="evenodd" d="M5 10a1 1 0 011-1h8a1 1 0 110 2H6a1 1 0 01-1-1z" clip-rule="evenodd" />
+              </svg>
+              No change
+            </span>
+          <% end %>
+        </div>
+      </div>
+      
+      <div class="comparison-column">
+        <div class="comparison-header">
+          <h4>Run 2</h4>
+          <span class="table__badge table__badge--info">v<%= @run2.prompt_version.version_number %></span>
+          <span class="text-muted"><%= @run2.created_at.strftime("%b %d, %Y %I:%M %p") %></span>
+        </div>
+        <div class="comparison-metrics">
+          <div class="metric">
+            <div class="metric__label">Success Rate</div>
+            <div class="metric__value metric__value--large <%= @success_rate_diff > 0 ? 'text-success' : @success_rate_diff < 0 ? 'text-danger' : '' %>">
+              <%= number_to_percentage(@run2_success_rate, precision: 1) %>
+            </div>
+          </div>
+          <div class="metric">
+            <div class="metric__label">Passed Tests</div>
+            <div class="metric__value">
+              <%= @run2.passed_count %> / <%= @run2.total_count %>
+            </div>
+          </div>
+          <div class="metric">
+            <div class="metric__label">Run Time</div>
+            <div class="metric__value">
+              <% if @run2.completed_at && @run2.started_at %>
+                <%= distance_of_time_in_words(@run2.started_at, @run2.completed_at) %>
+              <% else %>
+                —
+              <% end %>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Prompt Content Comparison -->
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Prompt Content Comparison</h3>
+  </div>
+  <div class="card__body">
+    <div class="comparison-grid">
+      <div class="comparison-column">
+        <div class="comparison-content">
+          <% if @run1.prompt_version.system_message.present? %>
+            <div class="comparison-section">
+              <h5 class="comparison-section__title">System Message</h5>
+              <pre class="prompt-content prompt-content--comparison"><%= @run1.prompt_version.system_message %></pre>
+            </div>
+          <% end %>
+          <div class="comparison-section">
+            <h5 class="comparison-section__title">Prompt Content</h5>
+            <pre class="prompt-content prompt-content--comparison"><%= @run1.prompt_version.content %></pre>
+          </div>
+        </div>
+      </div>
+      
+      <div class="comparison-divider"></div>
+      
+      <div class="comparison-column">
+        <div class="comparison-content">
+          <% if @run2.prompt_version.system_message.present? %>
+            <div class="comparison-section">
+              <h5 class="comparison-section__title">System Message</h5>
+              <pre class="prompt-content prompt-content--comparison"><%= @run2.prompt_version.system_message %></pre>
+            </div>
+          <% end %>
+          <div class="comparison-section">
+            <h5 class="comparison-section__title">Prompt Content</h5>
+            <pre class="prompt-content prompt-content--comparison"><%= @run2.prompt_version.content %></pre>
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Model Configuration Comparison -->
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Model Configuration</h3>
+  </div>
+  <div class="card__body">
+    <div class="comparison-grid">
+      <div class="comparison-column">
+        <dl class="comparison-list">
+          <dt>Model</dt>
+          <dd><%= @run1.prompt_version.model || "Default" %></dd>
+          
+          <dt>Temperature</dt>
+          <dd><%= @run1.prompt_version.temperature || "Default" %></dd>
+          
+          <dt>Max Tokens</dt>
+          <dd><%= @run1.prompt_version.max_tokens || "Default" %></dd>
+        </dl>
+      </div>
+      
+      <div class="comparison-divider"></div>
+      
+      <div class="comparison-column">
+        <dl class="comparison-list">
+          <dt>Model</dt>
+          <dd class="<%= @run1.prompt_version.model != @run2.prompt_version.model ? 'text-warning' : '' %>">
+            <%= @run2.prompt_version.model || "Default" %>
+          </dd>
+          
+          <dt>Temperature</dt>
+          <dd class="<%= @run1.prompt_version.temperature != @run2.prompt_version.temperature ? 'text-warning' : '' %>">
+            <%= @run2.prompt_version.temperature || "Default" %>
+          </dd>
+          
+          <dt>Max Tokens</dt>
+          <dd class="<%= @run1.prompt_version.max_tokens != @run2.prompt_version.max_tokens ? 'text-warning' : '' %>">
+            <%= @run2.prompt_version.max_tokens || "Default" %>
+          </dd>
+        </dl>
+      </div>
+    </div>
+  </div>
+</div>
+
+<!-- Test Results Summary -->
+<div class="card">
+  <div class="card__header">
+    <h3 class="card__title">Test Results Summary</h3>
+  </div>
+  <div class="card__body">
+    <div class="alert alert--info mb-md">
+      <p>
+        Individual test case results are available in the OpenAI evaluation reports:
+      </p>
+      <div class="mt-md">
+        <% if @run1.report_url.present? %>
+          <%= link_to "View Run 1 Report", @run1.report_url, 
+              target: "_blank", rel: "noopener", class: "btn btn--secondary btn--small" %>
+        <% end %>
+        <% if @run2.report_url.present? %>
+          <%= link_to "View Run 2 Report", @run2.report_url, 
+              target: "_blank", rel: "noopener", class: "btn btn--secondary btn--small" %>
+        <% end %>
+      </div>
+    </div>
+    
+    <p class="text-muted">
+      The OpenAI evaluation reports provide detailed test-by-test comparisons, including actual outputs 
+      and specific failure reasons for each test case.
+    </p>
+  </div>
+</div>

data/app/views/prompt_engine/eval_sets/edit.html.erb

@@ -0,0 +1,111 @@
+<div class="admin-header">
+  <div>
+    <h1>Edit Evaluation Set</h1>
+    <p class="text-muted">Update evaluation set details</p>
+  </div>
+</div>
+
+<div class="card">
+  <div class="card__body">
+    <%= form_with model: [@prompt, @eval_set], url: prompt_eval_set_path(@prompt, @eval_set), local: true do |form| %>
+      <%= render 'prompt_engine/shared/form_errors', object: @eval_set %>
+
+      <div class="form__group">
+        <%= form.label :name, class: "form__label" %>
+        <%= form.text_field :name, class: "form__input", required: true %>
+        <p class="form__help">A descriptive name for this evaluation set</p>
+      </div>
+
+      <div class="form__group">
+        <%= form.label :description, class: "form__label" %>
+        <%= form.text_area :description, class: "form__input", rows: 3 %>
+        <p class="form__help">Optional description of what this evaluation set tests</p>
+      </div>
+
+      <div class="form__group">
+        <%= form.label :grader_type, "Grader Type", class: "form__label" %>
+        <span class="form__required">*</span>
+        <%= form.select :grader_type,
+            options_for_select(PromptEngine::EvalSet::GRADER_TYPES.map { |k, v| [v, k] }, @eval_set.grader_type),
+            { include_blank: false },
+            { class: "form__input", id: "grader-type-select",
+              data: { controller: "grader-config", action: "change->grader-config#toggleConfig" } } %>
+        <p class="form__help">Select how the expected output should be compared to the actual output</p>
+      </div>
+
+      <div id="grader-config-fields" class="form__group" style="display: none;">
+        <label class="form__label">Grader Configuration</label>
+
+        <div id="regex-config" class="grader-config-section" style="display: none;">
+          <%= form.label :grader_config_pattern, "Regular Expression Pattern", class: "form__label" %>
+          <span class="form__required">*</span>
+          <%= text_field_tag "eval_set[grader_config][pattern]",
+              @eval_set.grader_config&.dig('pattern'),
+              class: "form__input",
+              placeholder: "e.g., ^Hello.*world$",
+              data: { grader_config_field: "regex" } %>
+          <p class="form__help">Enter a valid regular expression pattern to match against the output</p>
+        </div>
+
+        <div id="json-schema-config" class="grader-config-section" style="display: none;">
+          <%= form.label :grader_config_schema, "JSON Schema", class: "form__label" %>
+          <span class="form__required">*</span>
+          <%= text_area_tag "eval_set[grader_config][schema]",
+              @eval_set.grader_config&.dig('schema')&.to_json,
+              rows: 8,
+              class: "form__input",
+              placeholder: '{ "type": "object", "properties": { "name": { "type": "string" } }, "required": ["name"] }',
+              data: { grader_config_field: "json_schema" } %>
+          <p class="form__help">Note: Currently validates exact JSON match. Full schema validation coming soon.</p>
+        </div>
+      </div>
+
+      <div class="form__actions">
+        <%= form.submit "Update Evaluation Set", class: "btn btn--primary btn--medium",
+            data: { disable_with: "Updating..." } %>
+        <%= link_to "Cancel", prompt_eval_set_path(@prompt, @eval_set), class: "btn btn--secondary btn--medium" %>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<div class="mt-lg">
+  <%= button_to "Delete Evaluation Set", prompt_eval_set_path(@prompt, @eval_set), 
+      method: :delete,
+      data: { confirm: "Are you sure you want to delete this evaluation set? This will also delete all test cases and evaluation runs." },
+      class: "btn btn--danger btn--medium" %>
+</div>
+
+<script>
+  document.addEventListener('DOMContentLoaded', function() {
+    const graderTypeSelect = document.getElementById('grader-type-select');
+    const graderConfigFields = document.getElementById('grader-config-fields');
+    const regexConfig = document.getElementById('regex-config');
+    const jsonSchemaConfig = document.getElementById('json-schema-config');
+
+    function toggleGraderConfig() {
+      const selectedGrader = graderTypeSelect.value;
+
+      // Hide all config sections first
+      regexConfig.style.display = 'none';
+      jsonSchemaConfig.style.display = 'none';
+
+      // Show appropriate config section based on grader type
+      if (selectedGrader === 'regex') {
+        graderConfigFields.style.display = 'block';
+        regexConfig.style.display = 'block';
+      } else if (selectedGrader === 'json_schema') {
+        graderConfigFields.style.display = 'block';
+        jsonSchemaConfig.style.display = 'block';
+      } else {
+        graderConfigFields.style.display = 'none';
+      }
+    }
+
+    // Add event listener
+    graderTypeSelect.addEventListener('change', toggleGraderConfig);
+
+    // Initialize on page load
+    toggleGraderConfig();
+  });
+</script>

data/app/views/prompt_engine/eval_sets/index.html.erb

@@ -0,0 +1,63 @@
+<div class="admin-header">
+  <div>
+    <h1>Evaluation Sets for <%= @prompt.name %></h1>
+    <p class="text-muted">Manage test cases and run evaluations to measure prompt performance</p>
+  </div>
+  <div class="btn-group">
+    <%= link_to "Back to Prompt", prompt_path(@prompt), class: "btn btn--secondary btn--medium" %>
+    <%= link_to "New Evaluation Set", new_prompt_eval_set_path(@prompt), class: "btn btn--primary btn--medium" %>
+  </div>
+</div>
+
+<div class="card">
+  <div class="card__body">
+    <% if @eval_sets.any? %>
+      <div class="table-container">
+        <table class="table">
+          <thead>
+            <tr>
+              <th>Name</th>
+              <th>Description</th>
+              <th>Test Cases</th>
+              <th>Last Run</th>
+              <th class="table__actions">Actions</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @eval_sets.each do |eval_set| %>
+              <tr>
+                <td>
+                  <div class="table__primary">
+                    <%= link_to eval_set.name, prompt_eval_set_path(@prompt, eval_set), class: "table__link" %>
+                  </div>
+                </td>
+                <td>
+                  <span class="table__secondary"><%= eval_set.description || "—" %></span>
+                </td>
+                <td>
+                  <span class="table__secondary"><%= pluralize(eval_set.test_cases.count, 'test case') %></span>
+                </td>
+                <td>
+                  <% if eval_set.eval_runs.any? %>
+                    <span class="table__secondary"><%= time_ago_in_words(eval_set.eval_runs.last.created_at) %> ago</span>
+                  <% else %>
+                    <span class="table__secondary text-muted">Never run</span>
+                  <% end %>
+                </td>
+                <td class="table__actions">
+                  <%= link_to "View", prompt_eval_set_path(@prompt, eval_set), class: "btn btn--secondary btn--small" %>
+                  <%= link_to "Edit", edit_prompt_eval_set_path(@prompt, eval_set), class: "btn btn--secondary btn--small" %>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    <% else %>
+      <div class="table-empty">
+        <p class="text-muted mb-md">No evaluation sets created yet.</p>
+        <%= link_to "Create Your First Evaluation Set", new_prompt_eval_set_path(@prompt), class: "btn btn--primary btn--medium" %>
+      </div>
+    <% end %>
+  </div>
+</div>