prompt_engine 1.0.0

data/app/views/prompt_engine/eval_sets/show.html.erb

@@ -0,0 +1,235 @@
+<div class="admin-header">
+  <div>
+    <h1><%= @eval_set.name %></h1>
+    <% if @eval_set.description.present? %>
+      <p class="text-muted"><%= @eval_set.description %></p>
+    <% end %>
+    <p class="text-muted">
+      <strong>Grader Type:</strong> <%= @eval_set.grader_type_display %>
+      <% if @eval_set.grader_type == 'regex' && @eval_set.grader_config['pattern'].present? %>
+        <br><strong>Pattern:</strong> <code><%= @eval_set.grader_config['pattern'] %></code>
+      <% elsif @eval_set.grader_type == 'json_schema' && @eval_set.grader_config['schema'].present? %>
+        <br><strong>Schema:</strong> <code><%= @eval_set.grader_config['schema'].to_json.truncate(100) %></code>
+      <% end %>
+    </p>
+  </div>
+  <div class="btn-group">
+    <%= link_to "Back to Eval Sets", prompt_eval_sets_path(@prompt), class: "btn btn--secondary btn--medium" %>
+    <% if @recent_runs.any? %>
+      <%= link_to "View Metrics", metrics_prompt_eval_set_path(@prompt, @eval_set), class: "btn btn--secondary btn--medium" %>
+    <% end %>
+    <%= link_to "Add Test Case", new_prompt_eval_set_test_case_path(@prompt, @eval_set), class: "btn btn--secondary btn--medium" %>
+    <% if controller.send(:api_key_configured?) %>
+      <%= button_to "Run Evaluation", run_prompt_eval_set_path(@prompt, @eval_set), 
+          method: :post, 
+          class: "btn btn--primary btn--medium",
+          id: "run-evaluation-btn",
+          data: { 
+            disable_with: "Running Evaluation...", 
+            confirm: @test_cases.any? ? nil : "No test cases defined. Are you sure you want to run an empty evaluation?" 
+          } %>
+    <% else %>
+      <%= link_to "Configure API Key", settings_path, 
+          class: "btn btn--primary btn--medium",
+          title: "OpenAI API key required to run evaluations" %>
+    <% end %>
+    <%= link_to "Edit", edit_prompt_eval_set_path(@prompt, @eval_set), class: "btn btn--secondary btn--medium" %>
+  </div>
+</div>
+
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Test Cases</h3>
+    <div class="card__actions">
+      <%= link_to "Import", import_prompt_eval_set_test_cases_path(@prompt, @eval_set), class: "btn btn--secondary btn--small" %>
+      <%= link_to "Add Test Case", new_prompt_eval_set_test_case_path(@prompt, @eval_set), class: "btn btn--secondary btn--small" %>
+    </div>
+  </div>
+  <div class="card__body">
+    <% if @test_cases.any? %>
+      <div class="table-container">
+        <table class="table">
+          <thead>
+            <tr>
+              <th>Description</th>
+              <th>Input Variables</th>
+              <th>Expected Output</th>
+              <th class="table__actions">Actions</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @test_cases.each do |test_case| %>
+              <tr>
+                <td>
+                  <div class="table__primary">
+                    <%= test_case.description || "Test case ##{test_case.id}" %>
+                  </div>
+                </td>
+                <td>
+                  <code class="code-inline">
+                    <%= truncate(test_case.input_variables.to_json, length: 100) %>
+                  </code>
+                </td>
+                <td>
+                  <code class="code-inline">
+                    <%= truncate(test_case.expected_output, length: 100) %>
+                  </code>
+                </td>
+                <td class="table__actions">
+                  <%= link_to "Edit", edit_prompt_eval_set_test_case_path(@prompt, @eval_set, test_case), 
+                      class: "btn btn--secondary btn--small" %>
+                  <%= button_to "Delete", prompt_eval_set_test_case_path(@prompt, @eval_set, test_case), 
+                      method: :delete, 
+                      data: { confirm: "Are you sure you want to delete this test case?" }, 
+                      class: "btn btn--danger btn--small" %>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    <% else %>
+      <div class="table-empty">
+        <p class="text-muted">No test cases added yet.</p>
+        <div class="btn-group">
+          <%= link_to "Import Test Cases", import_prompt_eval_set_test_cases_path(@prompt, @eval_set), 
+              class: "btn btn--secondary btn--medium" %>
+          <%= link_to "Add Your First Test Case", new_prompt_eval_set_test_case_path(@prompt, @eval_set), 
+              class: "btn btn--primary btn--medium" %>
+        </div>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<div class="card">
+  <div class="card__header">
+    <h3 class="card__title">Recent Evaluation Runs</h3>
+    <% if @recent_runs.select { |r| r.status == 'completed' }.count >= 2 %>
+      <div class="card__actions">
+        <button type="submit" form="compare-runs-form" class="btn btn--secondary btn--small" 
+                id="compare-runs-btn" disabled>
+          Compare Selected (0/2)
+        </button>
+      </div>
+    <% end %>
+  </div>
+  <div class="card__body">
+    <% if @recent_runs.any? %>
+      <%= form_with url: compare_prompt_eval_set_path(@prompt, @eval_set), method: :get, 
+                    local: true, id: "compare-runs-form" do |form| %>
+        <div class="table-container">
+          <table class="table">
+            <thead>
+              <tr>
+                <% if @recent_runs.select { |r| r.status == 'completed' }.count >= 2 %>
+                  <th class="table__checkbox-column">Compare</th>
+                <% end %>
+                <th>Run Date</th>
+                <th>Prompt Version</th>
+                <th>Status</th>
+                <th>Results</th>
+                <th class="table__actions">Actions</th>
+              </tr>
+            </thead>
+            <tbody>
+            <% @recent_runs.each do |run| %>
+              <tr>
+                <% if @recent_runs.select { |r| r.status == 'completed' }.count >= 2 %>
+                  <td class="table__checkbox-column">
+                    <% if run.status == 'completed' %>
+                      <%= check_box_tag "run_ids[]", run.id, false, 
+                          class: "compare-checkbox",
+                          data: { 
+                            run_id: run.id,
+                            version: "v#{run.prompt_version.version_number}",
+                            date: run.created_at.strftime("%b %d, %Y %I:%M %p")
+                          } %>
+                    <% else %>
+                      <span class="text-muted" title="Only completed runs can be compared">—</span>
+                    <% end %>
+                  </td>
+                <% end %>
+                <td>
+                  <div class="table__primary">
+                    <%= run.created_at.strftime("%b %d, %Y %I:%M %p") %>
+                  </div>
+                  <span class="table__secondary"><%= time_ago_in_words(run.created_at) %> ago</span>
+                </td>
+                <td>
+                  <span class="table__badge table__badge--info">v<%= run.prompt_version.version_number %></span>
+                </td>
+                <td>
+                  <span class="table__badge table__badge--<%= run.status %>">
+                    <%= run.status.humanize %>
+                  </span>
+                </td>
+                <td>
+                  <% if run.total_count > 0 %>
+                    <div class="table__metric">
+                      <span class="table__metric-value"><%= run.passed_count %> / <%= run.total_count %></span>
+                      <span class="table__metric-unit">passed</span>
+                    </div>
+                    <span class="table__secondary">
+                      <%= number_to_percentage((run.passed_count.to_f / run.total_count * 100), precision: 0) %> success rate
+                    </span>
+                  <% else %>
+                    <span class="table__secondary text-muted">—</span>
+                  <% end %>
+                </td>
+                <td class="table__actions">
+                  <%= link_to "View Results", prompt_eval_run_path(@prompt, run), 
+                      class: "btn btn--secondary btn--small" %>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+      <% end %>
+    <% else %>
+      <div class="table-empty">
+        <p class="text-muted">No evaluation runs yet.</p>
+        <% if @test_cases.any? %>
+          <%= button_to "Run Your First Evaluation", run_prompt_eval_set_path(@prompt, @eval_set), 
+              method: :post, class: "btn btn--primary btn--medium" %>
+        <% else %>
+          <p class="text-muted">Add test cases before running evaluations.</p>
+        <% end %>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<% if @recent_runs.select { |r| r.status == 'completed' }.count >= 2 %>
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+  const checkboxes = document.querySelectorAll('.compare-checkbox');
+  const compareBtn = document.getElementById('compare-runs-btn');
+  
+  function updateCompareButton() {
+    const checkedBoxes = document.querySelectorAll('.compare-checkbox:checked');
+    const count = checkedBoxes.length;
+    
+    compareBtn.disabled = count !== 2;
+    compareBtn.textContent = `Compare Selected (${count}/2)`;
+    
+    // Disable unchecked checkboxes if 2 are already selected
+    checkboxes.forEach(checkbox => {
+      if (!checkbox.checked && count === 2) {
+        checkbox.disabled = true;
+      } else {
+        checkbox.disabled = false;
+      }
+    });
+  }
+  
+  checkboxes.forEach(checkbox => {
+    checkbox.addEventListener('change', updateCompareButton);
+  });
+  
+  // Initialize button state
+  updateCompareButton();
+});
+</script>
+<% end %>

data/app/views/prompt_engine/evaluations/index.html.erb

@@ -0,0 +1,194 @@
+<div class="admin-header">
+  <div>
+    <h1>Evaluations</h1>
+    <p class="text-muted">Manage evaluation sets and view results across all prompts</p>
+  </div>
+</div>
+
+<!-- Summary Cards -->
+<div class="metrics-summary mb-lg">
+  <div class="metrics-card">
+    <div class="metrics-card__icon">
+      <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+        <path d="M9 5H7a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2V7a2 2 0 00-2-2h-2M9 5a2 2 0 002 2h2a2 2 0 002-2M9 5a2 2 0 012-2h2a2 2 0 012 2"></path>
+      </svg>
+    </div>
+    <div class="metrics-card__content">
+      <h3 class="metrics-card__label">Evaluation Sets</h3>
+      <p class="metrics-card__value"><%= @total_eval_sets %></p>
+    </div>
+  </div>
+
+  <div class="metrics-card">
+    <div class="metrics-card__icon">
+      <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+        <path d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z"></path>
+      </svg>
+    </div>
+    <div class="metrics-card__content">
+      <h3 class="metrics-card__label">Total Runs</h3>
+      <p class="metrics-card__value"><%= @total_eval_runs %></p>
+    </div>
+  </div>
+
+  <div class="metrics-card">
+    <div class="metrics-card__icon">
+      <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+        <path d="M9 11l3 3L22 4"></path>
+        <path d="M21 12v7a2 2 0 01-2 2H5a2 2 0 01-2-2V5a2 2 0 012-2h11"></path>
+      </svg>
+    </div>
+    <div class="metrics-card__content">
+      <h3 class="metrics-card__label">Test Cases</h3>
+      <p class="metrics-card__value"><%= @total_test_cases %></p>
+    </div>
+  </div>
+
+  <div class="metrics-card">
+    <div class="metrics-card__icon">
+      <svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
+        <path d="M13 7h8m0 0v8m0-8l-8 8-4-4-6 6"></path>
+      </svg>
+    </div>
+    <div class="metrics-card__content">
+      <h3 class="metrics-card__label">Overall Pass Rate</h3>
+      <p class="metrics-card__value"><%= number_to_percentage(@overall_pass_rate, precision: 1) %></p>
+    </div>
+  </div>
+</div>
+
+<!-- Prompts with Evaluation Sets -->
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Prompts with Evaluations</h3>
+  </div>
+  <div class="card__body">
+    <% if @prompts_with_eval_sets.any? %>
+      <div class="eval-prompts-list">
+        <% @prompts_with_eval_sets.each do |prompt| %>
+          <div class="eval-prompt-item">
+            <div class="eval-prompt-header">
+              <h4 class="eval-prompt-title">
+                <%= link_to prompt.name, prompt_path(prompt), class: "table__link" %>
+              </h4>
+              <span class="table__badge table__badge--<%= prompt.status %>">
+                <%= prompt.status %>
+              </span>
+            </div>
+            
+            <div class="eval-sets-grid">
+              <% prompt.eval_sets.each do |eval_set| %>
+                <div class="eval-set-card">
+                  <div class="eval-set-card__header">
+                    <h5 class="eval-set-card__title">
+                      <%= link_to eval_set.name, prompt_eval_set_path(prompt, eval_set), class: "table__link" %>
+                    </h5>
+                    <div class="eval-set-card__stats">
+                      <span class="eval-set-card__stat">
+                        <%= pluralize(eval_set.test_cases.count, 'test') %>
+                      </span>
+                      <span class="eval-set-card__stat">
+                        <%= pluralize(eval_set.eval_runs.count, 'run') %>
+                      </span>
+                    </div>
+                  </div>
+                  
+                  <% if eval_set.description.present? %>
+                    <p class="eval-set-card__description">
+                      <%= truncate(eval_set.description, length: 100) %>
+                    </p>
+                  <% end %>
+                  
+                  <div class="eval-set-card__actions">
+                    <%= link_to "View", prompt_eval_set_path(prompt, eval_set), 
+                        class: "btn btn--secondary btn--small" %>
+                    <% if eval_set.eval_runs.where(status: 'completed').any? %>
+                      <%= link_to "Metrics", metrics_prompt_eval_set_path(prompt, eval_set), 
+                          class: "btn btn--secondary btn--small" %>
+                    <% end %>
+                  </div>
+                </div>
+              <% end %>
+            </div>
+          </div>
+        <% end %>
+      </div>
+    <% else %>
+      <div class="table-empty">
+        <p class="text-muted">No evaluation sets created yet.</p>
+        <p class="text-muted">Create evaluation sets from the prompt details page.</p>
+        <%= link_to "View Prompts", prompts_path, class: "btn btn--primary btn--medium mt-md" %>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<!-- Recent Activity -->
+<div class="card">
+  <div class="card__header">
+    <h3 class="card__title">Recent Evaluation Activity</h3>
+  </div>
+  <div class="card__body">
+    <% if @recent_runs.any? %>
+      <div class="table-container">
+        <table class="table table--simple">
+          <thead>
+            <tr>
+              <th>Date</th>
+              <th>Prompt</th>
+              <th>Evaluation Set</th>
+              <th>Status</th>
+              <th>Results</th>
+              <th class="table__actions">Actions</th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @recent_runs.each do |run| %>
+              <tr>
+                <td>
+                  <div class="table__primary">
+                    <%= run.created_at.strftime("%b %d, %Y %I:%M %p") %>
+                  </div>
+                </td>
+                <td>
+                  <%= link_to run.eval_set.prompt.name, prompt_path(run.eval_set.prompt), 
+                      class: "table__link" %>
+                </td>
+                <td>
+                  <%= link_to run.eval_set.name, prompt_eval_set_path(run.eval_set.prompt, run.eval_set), 
+                      class: "table__link" %>
+                </td>
+                <td>
+                  <span class="table__badge table__badge--<%= run.status %>">
+                    <%= run.status.humanize %>
+                  </span>
+                </td>
+                <td>
+                  <% if run.status == 'completed' && run.total_count > 0 %>
+                    <% success_rate = (run.passed_count.to_f / run.total_count * 100) %>
+                    <div class="table__metric">
+                      <span class="table__metric-value 
+                        <%= success_rate >= 80 ? 'text-success' : success_rate >= 60 ? 'text-warning' : 'text-danger' %>">
+                        <%= number_to_percentage(success_rate, precision: 1) %>
+                      </span>
+                    </div>
+                  <% else %>
+                    <span class="table__secondary text-muted">—</span>
+                  <% end %>
+                </td>
+                <td class="table__actions">
+                  <%= link_to "View", prompt_eval_run_path(run.eval_set.prompt, run), 
+                      class: "table__action" %>
+                </td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    <% else %>
+      <div class="table-empty">
+        <p class="text-muted">No evaluation runs yet.</p>
+      </div>
+    <% end %>
+  </div>
+</div>

data/app/views/prompt_engine/playground/result.html.erb

@@ -0,0 +1,58 @@
+<div class="admin-header">
+  <div>
+    <h1>Test Results: <%= @prompt.name %></h1>
+    <p class="text-muted">Prompt execution completed</p>
+  </div>
+</div>
+
+<% if @error.present? %>
+  <div class="card mb-lg">
+    <div class="card__header">
+      <h3 class="card__title text-danger">Error</h3>
+    </div>
+    <div class="card__body">
+      <p class="text-danger"><%= @error %></p>
+    </div>
+  </div>
+<% else %>
+  <div class="card mb-lg">
+    <div class="card__header">
+      <h3 class="card__title">Rendered Prompt</h3>
+    </div>
+    <div class="card__body">
+      <pre class="prompt-content"><%= @rendered_prompt %></pre>
+    </div>
+  </div>
+
+  <div class="card mb-lg">
+    <div class="card__header">
+      <h3 class="card__title">AI Response</h3>
+    </div>
+    <div class="card__body">
+      <pre class="prompt-content"><%= @response %></pre>
+    </div>
+  </div>
+
+  <div class="card mb-lg">
+    <div class="card__header">
+      <h3 class="card__title">Execution Details</h3>
+    </div>
+    <div class="card__body">
+      <div class="detail-grid">
+        <div class="detail-item">
+          <label class="detail-label">Execution Time</label>
+          <div class="detail-value"><%= @execution_time %> seconds</div>
+        </div>
+        <div class="detail-item">
+          <label class="detail-label">Tokens Used</label>
+          <div class="detail-value"><%= @token_count %></div>
+        </div>
+      </div>
+    </div>
+  </div>
+<% end %>
+
+<div class="form__actions">
+  <%= link_to "Try Again", playground_prompt_path(@prompt), class: "btn btn--primary btn--medium" %>
+  <%= link_to "Back to Prompt", prompt_path(@prompt), class: "btn btn--secondary btn--medium" %>
+</div>

data/app/views/prompt_engine/playground/show.html.erb

@@ -0,0 +1,129 @@
+<div class="admin-header">
+  <div>
+    <h1>Test Prompt: <%= @prompt.name %></h1>
+    <p class="text-muted">Test your prompt with real AI providers</p>
+  </div>
+</div>
+
+<div class="card">
+  <div class="card__header">
+    <h3 class="card__title">Playground Settings</h3>
+  </div>
+  <div class="card__body">
+    <%= form_with url: playground_prompt_path(@prompt), method: :post, local: true, html: { class: "form" } do |form| %>
+      <div class="form__group">
+        <%= form.label :provider, "AI Provider", class: "form__label form__label--required" %>
+        <% 
+          # Determine default provider based on prompt's model
+          default_provider = case @prompt.model
+                           when /claude/i, /anthropic/i
+                             "anthropic"
+                           when /gpt/i, /openai/i
+                             "openai"
+                           else
+                             nil
+                           end
+        %>
+        <%= form.select :provider,
+            options_for_select([
+              ["Anthropic (claude-3-5-sonnet-20241022)", "anthropic"],
+              ["OpenAI (gpt-4o)", "openai"]
+            ], default_provider),
+            { prompt: "Select a provider" },
+            class: "form__select", required: true %>
+        <div class="form__help">
+          <% if @prompt.model.present? %>
+            Prompt is configured for <%= @prompt.model %>
+          <% else %>
+            Select which AI provider to test with
+          <% end %>
+        </div>
+      </div>
+
+      <div class="form__group">
+        <%= form.label :api_key, "API Key", class: "form__label form__label--required" %>
+        <% 
+          # Get the appropriate API key based on selected provider
+          default_api_key = nil
+          api_key_configured = false
+          
+          if default_provider == "anthropic" && @settings.anthropic_configured?
+            default_api_key = @settings.anthropic_api_key
+            api_key_configured = true
+          elsif default_provider == "openai" && @settings.openai_configured?
+            default_api_key = @settings.openai_api_key
+            api_key_configured = true
+          end
+        %>
+        <%= form.password_field :api_key, 
+            class: "form__input", 
+            required: true,
+            value: default_api_key,
+            placeholder: api_key_configured ? "Using saved API key" : "Enter your API key",
+            data: { 
+              anthropic_key: @settings.anthropic_api_key,
+              openai_key: @settings.openai_api_key
+            } %>
+        <div class="form__help">
+          <% if api_key_configured %>
+            Using saved API key from settings. <%= link_to "Change in settings", edit_settings_path, class: "link" %>
+          <% else %>
+            Your API key will not be stored. <%= link_to "Save in settings", edit_settings_path, class: "link" %>
+          <% end %>
+        </div>
+      </div>
+
+      <% if @parameters.any? %>
+        <h3 class="form__section-title">Prompt Parameters</h3>
+        <div class="form__help mb-md">Fill in the values for each parameter:</div>
+        
+        <% @parameters.each do |parameter_name| %>
+          <div class="form__group">
+            <%= form.label "parameters[#{parameter_name}]", parameter_name.humanize.titleize, class: "form__label" %>
+            <%= form.text_area "parameters[#{parameter_name}]", 
+                class: "form__textarea",
+                rows: 3,
+                placeholder: "Enter value for #{parameter_name}" %>
+          </div>
+        <% end %>
+      <% end %>
+
+      <div class="form__actions">
+        <%= link_to "Cancel", prompt_path(@prompt), class: "btn btn--secondary btn--medium" %>
+        <%= form.submit "Test Prompt", class: "btn btn--primary btn--medium" %>
+      </div>
+    <% end %>
+  </div>
+</div>
+
+<script>
+  // Update API key when provider changes
+  document.addEventListener('DOMContentLoaded', function() {
+    const providerSelect = document.getElementById('provider');
+    const apiKeyField = document.getElementById('api_key');
+    const helpText = apiKeyField.parentElement.querySelector('.form__help');
+    
+    if (providerSelect && apiKeyField) {
+      const anthropicKey = apiKeyField.dataset.anthropicKey;
+      const openaiKey = apiKeyField.dataset.openaiKey;
+      
+      providerSelect.addEventListener('change', function() {
+        const selectedProvider = this.value;
+        
+        if (selectedProvider === 'anthropic' && anthropicKey) {
+          apiKeyField.value = anthropicKey;
+          apiKeyField.placeholder = 'Using saved API key';
+          helpText.innerHTML = 'Using saved API key from settings. <a href="<%= edit_settings_path %>" class="link">Change in settings</a>';
+        } else if (selectedProvider === 'openai' && openaiKey) {
+          apiKeyField.value = openaiKey;
+          apiKeyField.placeholder = 'Using saved API key';
+          helpText.innerHTML = 'Using saved API key from settings. <a href="<%= edit_settings_path %>" class="link">Change in settings</a>';
+        } else {
+          apiKeyField.value = '';
+          apiKeyField.placeholder = 'Enter your API key';
+          helpText.innerHTML = 'Your API key will not be stored. <a href="<%= edit_settings_path %>" class="link">Save in settings</a>';
+        }
+      });
+    }
+  });
+</script>