RubyGems - prompt_engine - Versions diffs - 1.0.0 - Mend

prompt_engine 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (97) hide show

checksums.yaml +7 -0
data/MIT-LICENSE +20 -0
data/README.md +67 -0
data/Rakefile +22 -0
data/app/assets/stylesheets/prompt_engine/application.css +22 -0
data/app/assets/stylesheets/prompt_engine/buttons.css +124 -0
data/app/assets/stylesheets/prompt_engine/cards.css +63 -0
data/app/assets/stylesheets/prompt_engine/comparison.css +244 -0
data/app/assets/stylesheets/prompt_engine/components/_test_runs.css +144 -0
data/app/assets/stylesheets/prompt_engine/dashboard.css +343 -0
data/app/assets/stylesheets/prompt_engine/evaluations.css +124 -0
data/app/assets/stylesheets/prompt_engine/forms.css +198 -0
data/app/assets/stylesheets/prompt_engine/foundation.css +182 -0
data/app/assets/stylesheets/prompt_engine/layout.css +75 -0
data/app/assets/stylesheets/prompt_engine/loading.css +229 -0
data/app/assets/stylesheets/prompt_engine/notifications.css +78 -0
data/app/assets/stylesheets/prompt_engine/overrides.css +42 -0
data/app/assets/stylesheets/prompt_engine/prompts.css +237 -0
data/app/assets/stylesheets/prompt_engine/sidebar.css +90 -0
data/app/assets/stylesheets/prompt_engine/tables.css +250 -0
data/app/assets/stylesheets/prompt_engine/utilities.css +52 -0
data/app/assets/stylesheets/prompt_engine/versions.css +370 -0
data/app/clients/prompt_engine/open_ai_evals_client.rb +135 -0
data/app/controllers/prompt_engine/admin/base_controller.rb +7 -0
data/app/controllers/prompt_engine/application_controller.rb +4 -0
data/app/controllers/prompt_engine/dashboard_controller.rb +24 -0
data/app/controllers/prompt_engine/eval_runs_controller.rb +23 -0
data/app/controllers/prompt_engine/eval_sets_controller.rb +200 -0
data/app/controllers/prompt_engine/evaluations_controller.rb +32 -0
data/app/controllers/prompt_engine/playground_controller.rb +57 -0
data/app/controllers/prompt_engine/playground_run_results_controller.rb +41 -0
data/app/controllers/prompt_engine/prompts_controller.rb +70 -0
data/app/controllers/prompt_engine/settings_controller.rb +28 -0
data/app/controllers/prompt_engine/test_cases_controller.rb +231 -0
data/app/controllers/prompt_engine/versions_controller.rb +90 -0
data/app/helpers/prompt_engine/application_helper.rb +4 -0
data/app/jobs/prompt_engine/application_job.rb +4 -0
data/app/mailers/prompt_engine/application_mailer.rb +6 -0
data/app/models/prompt_engine/application_record.rb +5 -0
data/app/models/prompt_engine/eval_result.rb +19 -0
data/app/models/prompt_engine/eval_run.rb +40 -0
data/app/models/prompt_engine/eval_set.rb +97 -0
data/app/models/prompt_engine/parameter.rb +126 -0
data/app/models/prompt_engine/parameter_parser.rb +39 -0
data/app/models/prompt_engine/playground_run_result.rb +20 -0
data/app/models/prompt_engine/prompt.rb +192 -0
data/app/models/prompt_engine/prompt_version.rb +72 -0
data/app/models/prompt_engine/setting.rb +45 -0
data/app/models/prompt_engine/test_case.rb +29 -0
data/app/services/prompt_engine/evaluation_runner.rb +258 -0
data/app/services/prompt_engine/playground_executor.rb +124 -0
data/app/services/prompt_engine/variable_detector.rb +97 -0
data/app/views/layouts/prompt_engine/admin.html.erb +65 -0
data/app/views/layouts/prompt_engine/application.html.erb +17 -0
data/app/views/prompt_engine/dashboard/index.html.erb +230 -0
data/app/views/prompt_engine/eval_runs/show.html.erb +204 -0
data/app/views/prompt_engine/eval_sets/compare.html.erb +229 -0
data/app/views/prompt_engine/eval_sets/edit.html.erb +111 -0
data/app/views/prompt_engine/eval_sets/index.html.erb +63 -0
data/app/views/prompt_engine/eval_sets/metrics.html.erb +371 -0
data/app/views/prompt_engine/eval_sets/new.html.erb +113 -0
data/app/views/prompt_engine/eval_sets/show.html.erb +235 -0
data/app/views/prompt_engine/evaluations/index.html.erb +194 -0
data/app/views/prompt_engine/playground/result.html.erb +58 -0
data/app/views/prompt_engine/playground/show.html.erb +129 -0
data/app/views/prompt_engine/playground_run_results/index.html.erb +99 -0
data/app/views/prompt_engine/playground_run_results/show.html.erb +123 -0
data/app/views/prompt_engine/prompts/_form.html.erb +224 -0
data/app/views/prompt_engine/prompts/edit.html.erb +9 -0
data/app/views/prompt_engine/prompts/index.html.erb +80 -0
data/app/views/prompt_engine/prompts/new.html.erb +9 -0
data/app/views/prompt_engine/prompts/show.html.erb +297 -0
data/app/views/prompt_engine/settings/edit.html.erb +93 -0
data/app/views/prompt_engine/shared/_form_errors.html.erb +16 -0
data/app/views/prompt_engine/test_cases/edit.html.erb +72 -0
data/app/views/prompt_engine/test_cases/import.html.erb +92 -0
data/app/views/prompt_engine/test_cases/import_preview.html.erb +103 -0
data/app/views/prompt_engine/test_cases/new.html.erb +79 -0
data/app/views/prompt_engine/versions/_version_card.html.erb +56 -0
data/app/views/prompt_engine/versions/compare.html.erb +82 -0
data/app/views/prompt_engine/versions/index.html.erb +96 -0
data/app/views/prompt_engine/versions/show.html.erb +98 -0
data/config/routes.rb +61 -0
data/db/migrate/20250124000001_create_eval_tables.rb +43 -0
data/db/migrate/20250124000002_add_open_ai_fields_to_evals.rb +11 -0
data/db/migrate/20250125000001_add_grader_fields_to_eval_sets.rb +8 -0
data/db/migrate/20250723161909_create_prompts.rb +17 -0
data/db/migrate/20250723184757_create_prompt_engine_versions.rb +24 -0
data/db/migrate/20250723203838_create_prompt_engine_parameters.rb +20 -0
data/db/migrate/20250724160623_create_prompt_engine_playground_run_results.rb +30 -0
data/db/migrate/20250724165118_create_prompt_engine_settings.rb +14 -0
data/lib/prompt_engine/engine.rb +25 -0
data/lib/prompt_engine/version.rb +3 -0
data/lib/prompt_engine.rb +33 -0
data/lib/tasks/active_prompt_tasks.rake +32 -0
data/lib/tasks/eval_demo.rake +149 -0
metadata +293 -0

data/app/views/layouts/prompt_engine/application.html.erb ADDED Viewed

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Prompt Engine Admin</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+    <%= yield :head %>
+    <%= stylesheet_link_tag    "prompt_engine/application", media: "all" %>
+  </head>
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/app/views/prompt_engine/dashboard/index.html.erb ADDED Viewed

@@ -0,0 +1,230 @@
+<div class="admin-header">
+  <div>
+    <h1>Dashboard</h1>
+    <p class="text-muted">Overview of your prompt management system</p>
+  </div>
+</div>
+<div class="dashboard">
+  <div class="dashboard__stats">
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--primary">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <path d="M12 20h9"/>
+          <path d="M16.5 3.5a2.121 2.121 0 0 1 3 3L7 19l-4 1 1-4L16.5 3.5z"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Total Prompts</h3>
+        <p class="stat-card__value"><%= @total_prompts %></p>
+      </div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--success">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <circle cx="12" cy="12" r="10"/>
+          <path d="m9 12 2 2 4-4"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Active Prompts</h3>
+        <p class="stat-card__value"><%= @prompt_engines %></p>
+      </div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--info">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <polygon points="13 2 3 14 12 14 11 22 21 10 12 10 13 2"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Test Runs</h3>
+        <p class="stat-card__value"><%= @total_test_runs %></p>
+      </div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--warning">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <path d="M22 2 11 13"/>
+          <path d="m22 2-7 20-4-9-9-4Z"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Tokens Used</h3>
+        <p class="stat-card__value"><%= number_with_delimiter(@total_tokens_used) %></p>
+      </div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--secondary">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <path d="M9 11l3 3L22 4"/>
+          <path d="M21 12v7a2 2 0 01-2 2H5a2 2 0 01-2-2V5a2 2 0 012-2h11"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Evaluation Sets</h3>
+        <p class="stat-card__value"><%= @total_eval_sets %></p>
+      </div>
+    </div>
+    <div class="stat-card">
+      <div class="stat-card__icon stat-card__icon--danger">
+        <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+          <path d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z"/>
+        </svg>
+      </div>
+      <div class="stat-card__content">
+        <h3 class="stat-card__title">Evaluation Runs</h3>
+        <p class="stat-card__value"><%= @total_eval_runs %></p>
+      </div>
+    </div>
+  </div>
+  <div class="dashboard__content">
+    <div class="dashboard__section">
+      <div class="card">
+        <div class="card__header">
+          <h2 class="card__title">Recent Prompts</h2>
+          <%= link_to "View all", prompts_path, class: "btn btn--sm btn--ghost" %>
+        </div>
+        <div class="card__body">
+          <% if @recent_prompts.any? %>
+            <div class="table-container table-container--compact">
+              <table class="table">
+                <thead>
+                  <tr class="table__row">
+                    <th class="table__cell table__cell--header">Name</th>
+                    <th class="table__cell table__cell--header">Status</th>
+                    <th class="table__cell table__cell--header">Parameters</th>
+                    <th class="table__cell table__cell--header">Updated</th>
+                    <th class="table__cell table__cell--header table__actions-header"></th>
+                  </tr>
+                </thead>
+                <tbody>
+                  <% @recent_prompts.each do |prompt| %>
+                    <tr class="table__row">
+                      <td class="table__cell">
+                        <%= link_to prompt.name, prompt_path(prompt), class: "table__action" %>
+                      </td>
+                      <td class="table__cell">
+                        <span class="table__badge table__badge--<%= prompt.status %>">
+                          <%= prompt.status.humanize %>
+                        </span>
+                      </td>
+                      <td class="table__cell">
+                        <span class="table__secondary"><%= prompt.parameters.count %> params</span>
+                      </td>
+                      <td class="table__cell">
+                        <span class="table__secondary"><%= time_ago_in_words(prompt.updated_at) %> ago</span>
+                      </td>
+                      <td class="table__cell table__cell--actions">
+                        <%= link_to "Edit", edit_prompt_path(prompt), class: "btn btn--sm btn--ghost" %>
+                      </td>
+                    </tr>
+                  <% end %>
+                </tbody>
+              </table>
+            </div>
+          <% else %>
+            <div class="empty-state">
+              <p class="text-muted">No prompts created yet</p>
+              <%= link_to "Create your first prompt", new_prompt_path, class: "btn btn--primary" %>
+            </div>
+          <% end %>
+        </div>
+      </div>
+    </div>
+    <div class="dashboard__section">
+      <div class="card">
+        <div class="card__header">
+          <h2 class="card__title">Recent Test Runs</h2>
+          <%= link_to "View all", playground_run_results_path, class: "btn btn--sm btn--ghost" %>
+        </div>
+        <div class="card__body">
+          <% if @recent_test_runs.any? %>
+            <div class="test-runs-list">
+              <% @recent_test_runs.each do |run| %>
+                <div class="test-run-item">
+                  <div class="test-run-item__main">
+                    <div class="test-run-item__header">
+                      <h4 class="test-run-item__title">
+                        <%= link_to run.prompt_version.prompt.name, playground_run_result_path(run), class: "link" %>
+                      </h4>
+                      <span class="test-run-item__date text-muted">
+                        <%= time_ago_in_words(run.created_at) %> ago
+                      </span>
+                    </div>
+                    <div class="test-run-item__content">
+                      <p class="test-run-item__model text-muted">
+                        <%= run.model %> •
+                        <%= number_with_delimiter(run.token_count || 0) %> tokens •
+                        <%= number_with_precision(run.execution_time * 1000, precision: 0) %>ms
+                      </p>
+                    </div>
+                  </div>
+                </div>
+              <% end %>
+            </div>
+          <% else %>
+            <div class="empty-state">
+              <p class="text-muted">No test runs yet</p>
+              <%= link_to "Test a prompt", prompts_path, class: "btn btn--primary" %>
+            </div>
+          <% end %>
+        </div>
+      </div>
+    </div>
+    <div class="dashboard__section">
+      <div class="card">
+        <div class="card__header">
+          <h2 class="card__title">Recent Evaluation Runs</h2>
+          <%= link_to "View all", evaluations_path, class: "btn btn--sm btn--ghost" %>
+        </div>
+        <div class="card__body">
+          <% if @recent_eval_runs.any? %>
+            <div class="test-runs-list">
+              <% @recent_eval_runs.each do |run| %>
+                <div class="test-run-item">
+                  <div class="test-run-item__main">
+                    <div class="test-run-item__header">
+                      <h4 class="test-run-item__title">
+                        <%= link_to run.eval_set.name, prompt_eval_run_path(run.eval_set.prompt, run), class: "link" %>
+                      </h4>
+                      <span class="test-run-item__date text-muted">
+                        <%= time_ago_in_words(run.created_at) %> ago
+                      </span>
+                    </div>
+                    <div class="test-run-item__content">
+                      <p class="test-run-item__model text-muted">
+                        <%= run.eval_set.prompt.name %> •
+                        <%= pluralize(run.total_count, 'test') %> •
+                        <% if run.total_count > 0 %>
+                          <span class="<%= (run.passed_count.to_f / run.total_count * 100) >= 80 ? 'text-success' : 'text-danger' %>">
+                            <%= number_to_percentage((run.passed_count.to_f / run.total_count * 100), precision: 1) %> passed
+                          </span>
+                        <% else %>
+                          <span class="text-muted">No tests</span>
+                        <% end %>
+                      </p>
+                    </div>
+                  </div>
+                </div>
+              <% end %>
+            </div>
+          <% else %>
+            <div class="empty-state">
+              <p class="text-muted">No evaluation runs yet</p>
+              <%= link_to "View evaluations", evaluations_path, class: "btn btn--primary" %>
+            </div>
+          <% end %>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/prompt_engine/eval_runs/show.html.erb ADDED Viewed

@@ -0,0 +1,204 @@
+<div class="admin-header">
+  <div>
+    <h1>Evaluation Run Results</h1>
+    <p class="text-muted">
+      <%= @eval_run.eval_set.name %> •
+      <%= @eval_run.created_at.strftime("%b %d, %Y at %I:%M %p") %>
+    </p>
+  </div>
+  <div class="btn-group">
+    <%= link_to "Back to Eval Set", prompt_eval_set_path(@prompt, @eval_run.eval_set),
+        class: "btn btn--secondary btn--medium" %>
+    <% if @eval_run.report_url.present? %>
+      <%= link_to "View OpenAI Report", @eval_run.report_url,
+          target: "_blank", rel: "noopener", class: "btn btn--secondary btn--medium" %>
+    <% end %>
+  </div>
+</div>
+<div class="card mb-lg">
+  <div class="card__header">
+    <h3 class="card__title">Evaluation Summary</h3>
+  </div>
+  <div class="card__body">
+    <div class="detail-grid">
+      <div class="detail-item">
+        <label class="detail-label">Status</label>
+        <div class="detail-value">
+          <span class="table__badge table__badge--<%= @eval_run.status %>">
+            <%= @eval_run.status.humanize %>
+          </span>
+        </div>
+      </div>
+      <div class="detail-item">
+        <label class="detail-label">Prompt Version</label>
+        <div class="detail-value">
+          <%= link_to "v#{@eval_run.prompt_version.version_number}",
+              prompt_version_path(@prompt, @eval_run.prompt_version),
+              class: "table__link" %>
+        </div>
+      </div>
+      <div class="detail-item">
+        <label class="detail-label">Total Tests</label>
+        <div class="detail-value"><%= @eval_run.total_count %></div>
+      </div>
+      <div class="detail-item">
+        <label class="detail-label">Passed</label>
+        <div class="detail-value text-success"><%= @eval_run.passed_count %></div>
+      </div>
+      <div class="detail-item">
+        <label class="detail-label">Failed</label>
+        <div class="detail-value text-danger"><%= @eval_run.failed_count %></div>
+      </div>
+      <div class="detail-item">
+        <label class="detail-label">Success Rate</label>
+        <div class="detail-value">
+          <% if @eval_run.total_count > 0 %>
+            <%= number_to_percentage((@eval_run.passed_count.to_f / @eval_run.total_count * 100),
+                precision: 1) %>
+          <% else %>
+            <span class="text-muted">—</span>
+          <% end %>
+        </div>
+      </div>
+      <% if @eval_run.started_at.present? %>
+        <div class="detail-item">
+          <label class="detail-label">Started At</label>
+          <div class="detail-value"><%= @eval_run.started_at.strftime("%b %d, %Y %I:%M %p") %></div>
+        </div>
+      <% end %>
+      <% if @eval_run.completed_at.present? %>
+        <div class="detail-item">
+          <label class="detail-label">Completed At</label>
+          <div class="detail-value"><%= @eval_run.completed_at.strftime("%b %d, %Y %I:%M %p") %></div>
+        </div>
+        <div class="detail-item">
+          <label class="detail-label">Duration</label>
+          <div class="detail-value">
+            <%= distance_of_time_in_words(@eval_run.started_at, @eval_run.completed_at) %>
+          </div>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>
+<% if @eval_run.status == 'pending' %>
+  <div class="card mb-lg">
+    <div class="card__body">
+      <div class="text-center py-lg">
+        <div class="mb-md">
+          <div class="spinner spinner--large"></div>
+        </div>
+        <h3 class="mb-md">Preparing Evaluation</h3>
+        <p class="text-muted">Setting up evaluation environment...</p>
+        <p class="text-muted">This page will refresh automatically.</p>
+      </div>
+    </div>
+  </div>
+  <script>
+    setTimeout(function() {
+      location.reload();
+    }, 3000);
+  </script>
+<% elsif @eval_run.status == 'running' %>
+  <div class="card mb-lg">
+    <div class="card__body">
+      <div class="text-center py-lg">
+        <div class="mb-md">
+          <div class="spinner spinner--large"></div>
+        </div>
+        <h3 class="mb-md">Evaluation in Progress</h3>
+        <p class="text-muted">Running <%= @eval_run.eval_set.test_cases.count %> test cases on OpenAI's infrastructure...</p>
+        <p class="text-muted">This may take a few minutes depending on the number of test cases.</p>
+        <div class="mt-lg">
+          <div class="progress progress--striped progress--animated" style="max-width: 400px; margin: 0 auto;">
+            <div class="progress__bar" style="width: 50%"></div>
+          </div>
+        </div>
+        <p class="text-muted mt-md">This page will refresh automatically every 5 seconds.</p>
+      </div>
+    </div>
+  </div>
+  <script>
+    setTimeout(function() {
+      location.reload();
+    }, 5000);
+  </script>
+<% elsif @eval_run.status == 'failed' %>
+  <div class="card mb-lg">
+    <div class="card__body">
+      <div class="alert alert--danger">
+        <h3 class="mb-md">Evaluation Failed</h3>
+        <p><%= @eval_run.error_message || "An unknown error occurred during evaluation." %></p>
+        <% if @eval_run.error_message&.include?("API key") %>
+          <div class="mt-md">
+            <p><strong>How to fix:</strong></p>
+            <ul>
+              <li>Check your OpenAI API key configuration in <%= link_to "Settings", settings_path %></li>
+              <li>Ensure your API key has access to the Evals API</li>
+              <li>Verify your API key is not expired or revoked</li>
+            </ul>
+          </div>
+        <% elsif @eval_run.error_message&.include?("rate limit") %>
+          <div class="mt-md">
+            <p><strong>How to fix:</strong></p>
+            <ul>
+              <li>Wait a few minutes before trying again</li>
+              <li>Consider reducing the number of test cases</li>
+              <li>Check your OpenAI API usage limits</li>
+            </ul>
+          </div>
+        <% end %>
+        <div class="mt-lg">
+          <%= link_to "Back to Evaluation Set", prompt_eval_set_path(@prompt, @eval_run.eval_set),
+              class: "btn btn--secondary btn--medium" %>
+          <%= link_to "View Settings", settings_path, class: "btn btn--secondary btn--medium" %>
+        </div>
+      </div>
+    </div>
+  </div>
+<% elsif @eval_run.status == 'completed' %>
+  <div class="card">
+    <div class="card__header">
+      <h3 class="card__title">Results Overview</h3>
+    </div>
+    <div class="card__body">
+      <% if @eval_run.passed_count == @eval_run.total_count %>
+        <div class="alert alert--success mb-md">
+          <strong>All tests passed!</strong> Every test case produced the expected output.
+        </div>
+      <% elsif @eval_run.failed_count > 0 %>
+        <div class="alert alert--warning mb-md">
+          <strong><%= pluralize(@eval_run.failed_count, 'test') %> failed.</strong>
+          Some test cases did not produce the expected output.
+        </div>
+      <% end %>
+      <p class="text-muted">
+        Individual test results are available in the
+        <% if @eval_run.report_url.present? %>
+          <%= link_to "OpenAI evaluation report", @eval_run.report_url,
+              target: "_blank", rel: "noopener", class: "table__link" %>.
+        <% else %>
+          OpenAI evaluation report.
+        <% end %>
+        The report includes detailed information about each test case, including the actual outputs
+        and exact comparison results.
+      </p>
+    </div>
+  </div>
+<% end %>