leva 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e16081b74e57cb4f198b075e76bc5fc1c07f67e4a14c4d38e27f8df53e26f3cf
-  data.tar.gz: db9725fdc26fe3542ba8a1b1aaf2f22c2471e112c841224ce3587d1b6e83269c
+  metadata.gz: 63fc046c15809b57e461d3c7b8a35635b6a32c439b58d916e4d41e33e0f72827
+  data.tar.gz: 0e7f080fcb722d9f9539a139fecc7e0692a202e4e0bc8266f7f4b637750b2942
 SHA512:
-  metadata.gz: e1019c0ec416f40854b5b9ef7c4e38df0141c98c5684f01ed26ea145deba92151a0ed908936768819eb1ab11167f2e5b9e075389ef8e8b25fb84786c1571f64d
-  data.tar.gz: 9de9c5e4ee512a66c8b6a72f5b5d8d7b6b316ddf4be13652df640433a40bf6a06b207cce38ef4bc8631619880d9587f3318a77a8722cee394e617c8fc2995039
+  metadata.gz: fa195af952f524ff1d3b33106dc9dae8a58a9f24afeb658395ca255b485ccbcfb6cb78e48f00dbbba0f40e467afe8553ed795f1e3c67391e1b0cf2eebbf65a5c
+  data.tar.gz: 2da6f65c79f5e46f70033c24a3f7b30b1a873bd986bcefc556186ffc2b3dd38aae9036a9cb1ec375c480c06233f2fb99b713af6afdc0a4d0d12db0034196362e

data/README.md

@@ -1,6 +1,7 @@
 # Leva - Flexible Evaluation Framework for Language Models
 
 [![Gem Version](https://badge.fury.io/rb/leva.svg)](https://badge.fury.io/rb/leva)
+[![CI](https://github.com/kieranklaassen/leva/actions/workflows/ci.yml/badge.svg)](https://github.com/kieranklaassen/leva/actions/workflows/ci.yml)
 
 Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets on production models. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic on production data with security in mind.
 
@@ -28,13 +29,25 @@ rails leva:install:migrations
 rails db:migrate
 ```
 
+Mount the Leva engine in your application's routes file:
+
+```ruby
+# config/routes.rb
+Rails.application.routes.draw do
+  mount Leva::Engine => "/leva"
+  # your other routes...
+end
+```
+
+The Leva UI will then be available at `/leva` in your application.
+
 ## Usage
 
 ### 1. Setting up Datasets
 
 First, create a dataset and add any ActiveRecord records you want to evaluate against. To make your models compatible with Leva, include the `Leva::Recordable` concern in your model:
 
-````ruby
+```ruby
 class TextContent < ApplicationRecord
   include Leva::Recordable
 
@@ -71,7 +84,11 @@ class TextContent < ApplicationRecord
   end
 end
 
-dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset") dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive") dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative") dataset.add_record TextContent.create(text: "It's ok", expected_label: "Neutral")
+dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset")
+dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive")
+dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative")
+dataset.add_record TextContent.create(text: "It's ok", expected_label: "Neutral")
+```
 
 ### 2. Implementing Runs
 
@@ -79,7 +96,7 @@ Create a run class to handle the execution of your inference logic:
 
 ```bash
 rails generate leva:runner sentiment
-````
+```
 
 ```ruby
 class SentimentRun < Leva::BaseRun

data/app/controllers/leva/application_controller.rb

@@ -4,4 +4,4 @@ module Leva
   class ApplicationController < ActionController::Base
     protect_from_forgery with: :exception
   end
-end
+end

data/app/controllers/leva/dataset_records_controller.rb

@@ -18,4 +18,4 @@ module Leva
       @dataset = Dataset.find(params[:dataset_id])
     end
   end
-end
+end

data/app/controllers/leva/datasets_controller.rb

@@ -2,7 +2,7 @@
 
 module Leva
   class DatasetsController < ApplicationController
-    before_action :set_dataset, only: [:show, :edit, :update, :destroy]
+    before_action :set_dataset, only: [ :show, :edit, :update, :destroy ]
 
     # GET /datasets
     # @return [void]
@@ -35,7 +35,7 @@ module Leva
       @dataset = Dataset.new(dataset_params)
 
       if @dataset.save
-        redirect_to @dataset, notice: 'Dataset was successfully created.'
+        redirect_to @dataset, notice: "Dataset was successfully created."
       else
         render :new
       end
@@ -45,7 +45,7 @@ module Leva
     # @return [void]
     def update
       if @dataset.update(dataset_params)
-        redirect_to @dataset, notice: 'Dataset was successfully updated.'
+        redirect_to @dataset, notice: "Dataset was successfully updated."
       else
         render :edit
       end
@@ -55,10 +55,10 @@ module Leva
     # @return [void]
     def destroy
       if @dataset.dataset_records.any?
-        redirect_to @dataset, alert: 'Cannot delete dataset with existing records.'
+        redirect_to @dataset, alert: "Cannot delete dataset with existing records."
       else
         @dataset.destroy
-        redirect_to datasets_url, notice: 'Dataset was successfully destroyed.'
+        redirect_to datasets_url, notice: "Dataset was successfully destroyed."
       end
     end
 
@@ -76,4 +76,4 @@ module Leva
       params.require(:dataset).permit(:name, :description)
     end
   end
-end
+end

data/app/controllers/leva/experiments_controller.rb

@@ -4,9 +4,9 @@ module Leva
   class ExperimentsController < ApplicationController
     include ApplicationHelper
 
-    before_action :set_experiment, only: [:show, :edit, :update]
-    before_action :check_editable, only: [:edit, :update]
-    before_action :load_runners_and_evaluators, only: [:new, :edit, :create, :update]
+    before_action :set_experiment, only: [ :show, :edit, :update ]
+    before_action :check_editable, only: [ :edit, :update ]
+    before_action :load_runners_and_evaluators, only: [ :new, :edit, :create, :update ]
 
     # GET /experiments
     # @return [void]
@@ -39,7 +39,7 @@ module Leva
 
       if @experiment.save
         ExperimentJob.perform_later(@experiment) unless @experiment.completed?
-        redirect_to @experiment, notice: 'Experiment was successfully created and is now running.'
+        redirect_to @experiment, notice: "Experiment was successfully created and is now running."
       else
         render :new
       end
@@ -49,7 +49,7 @@ module Leva
     # @return [void]
     def update
       if @experiment.update(experiment_params)
-        redirect_to @experiment, notice: 'Experiment was successfully updated.'
+        redirect_to @experiment, notice: "Experiment was successfully updated."
       else
         render :edit
       end
@@ -69,7 +69,7 @@ module Leva
       # Queue the job again
       ExperimentJob.perform_later(@experiment)
 
-      redirect_to @experiment, notice: 'Experiment has been reset and is now running again.'
+      redirect_to @experiment, notice: "Experiment has been reset and is now running again."
     end
 
     private
@@ -92,7 +92,7 @@ module Leva
     end
 
     def check_editable
-      redirect_to @experiment, alert: 'Completed experiments cannot be edited.' if @experiment.completed?
+      redirect_to @experiment, alert: "Completed experiments cannot be edited." if @experiment.completed?
     end
   end
-end
+end

data/app/controllers/leva/runner_results_controller.rb

@@ -5,4 +5,4 @@ module Leva
       @runner_result = @experiment.runner_results.find(params[:id])
     end
   end
-end
+end

data/app/controllers/leva/workbench_controller.rb

@@ -4,9 +4,9 @@ module Leva
   class WorkbenchController < ApplicationController
     include ApplicationHelper
 
-    before_action :set_prompt, only: [:index, :edit, :update, :run, :run_all_evals, :run_evaluator]
-    before_action :set_dataset_record, only: [:index, :run, :run_all_evals, :run_evaluator]
-    before_action :set_runner_result, only: [:index, :run_all_evals, :run_evaluator]
+    before_action :set_prompt, only: [ :index, :edit, :update, :run, :run_all_evals, :run_evaluator ]
+    before_action :set_dataset_record, only: [ :index, :run, :run_all_evals, :run_evaluator ]
+    before_action :set_runner_result, only: [ :index, :run_all_evals, :run_evaluator ]
 
     # GET /workbench
     # @return [void]
@@ -17,6 +17,17 @@ module Leva
       @runners = load_runners
       @selected_runner = params[:runner] || @runners.first&.name
       @selected_dataset_record = params[:dataset_record_id] || DatasetRecord.first&.id
+
+      # Get merged context if runner and dataset record are available
+      if @selected_runner && @dataset_record
+        runner_class = @selected_runner.constantize rescue nil
+        if runner_class && runner_class < Leva::BaseRun
+          runner = runner_class.new
+          @record_context = @dataset_record.recordable.to_llm_context
+          @runner_context = runner.to_llm_context(@dataset_record.recordable)
+          @merged_context = @record_context.merge(@runner_context)
+        end
+      end
     end
 
     # GET /workbench/new
@@ -31,7 +42,7 @@ module Leva
     def create
       @prompt = Prompt.new(prompt_params)
       if @prompt.save
-        redirect_to workbench_index_path(prompt_id: @prompt.id), notice: 'Prompt was successfully created.'
+        redirect_to workbench_index_path(prompt_id: @prompt.id), notice: "Prompt was successfully created."
       else
         render :new
       end
@@ -47,45 +58,45 @@ module Leva
     def update
       @prompt = Prompt.find(params[:id])
       if @prompt.update(prompt_params)
-        render json: { status: 'success', message: 'Prompt updated successfully' }
+        render json: { status: "success", message: "Prompt updated successfully" }
       else
-        render json: { status: 'error', errors: @prompt.errors.full_messages }, status: :unprocessable_entity
+        render json: { status: "error", errors: @prompt.errors.full_messages }, status: :unprocessable_entity
       end
     end
 
     def run
-      return redirect_to workbench_index_path, alert: 'Please select a record and a runner' unless @dataset_record && run_params[:runner]
+      return redirect_to workbench_index_path, alert: "Please select a record and a runner" unless @dataset_record && run_params[:runner]
 
       runner_class = run_params[:runner].constantize
-      return redirect_to workbench_index_path, alert: 'Invalid runner selected' unless runner_class < Leva::BaseRun
+      return redirect_to workbench_index_path, alert: "Invalid runner selected" unless runner_class < Leva::BaseRun
 
       runner = runner_class.new
       runner_result = runner.execute_and_store(nil, @dataset_record, @prompt)
 
-      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: run_params[:runner]), notice: 'Run completed successfully'
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: run_params[:runner]), notice: "Run completed successfully"
     end
 
     def run_all_evals
-      return redirect_to workbench_index_path, alert: 'No runner result available' unless @runner_result
+      return redirect_to workbench_index_path, alert: "No runner result available" unless @runner_result
 
       load_evaluators.each do |evaluator_class|
         evaluator = evaluator_class.new
         evaluator.evaluate_and_store(nil, @runner_result)
       end
 
-      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: 'All evaluations completed successfully'
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: "All evaluations completed successfully"
     end
 
     def run_evaluator
-      return redirect_to workbench_index_path, alert: 'No runner result available' unless @runner_result
+      return redirect_to workbench_index_path, alert: "No runner result available" unless @runner_result
 
       evaluator_class = params[:evaluator].constantize
-      return redirect_to workbench_index_path, alert: 'Invalid evaluator selected' unless evaluator_class < Leva::BaseEval
+      return redirect_to workbench_index_path, alert: "Invalid evaluator selected" unless evaluator_class < Leva::BaseEval
 
       evaluator = evaluator_class.new
       evaluator.evaluate_and_store(nil, @runner_result)
 
-      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: 'Evaluator run successfully'
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: "Evaluator run successfully"
     end
 
     private
@@ -110,4 +121,4 @@ module Leva
       @runner_result = @dataset_record.runner_results.last if @dataset_record
     end
   end
-end
+end

data/app/helpers/leva/application_helper.rb

@@ -4,24 +4,24 @@ module Leva
     #
     # @return [Array<Class>] An array of evaluator classes
     def load_evaluators
-      load_classes_from_directory('app/evals', Leva::BaseEval) || []
+      load_classes_from_directory("app/evals", Leva::BaseEval) || []
     end
 
     # Loads all runner classes that inherit from Leva::BaseRun
     #
     # @return [Array<Class>] An array of runner classes
     def load_runners
-      load_classes_from_directory('app/runners', Leva::BaseRun) || []
+      load_classes_from_directory("app/runners", Leva::BaseRun) || []
     end
 
     # Loads predefined prompts from markdown files
     #
     # @return [Array<Array<String, String>>] An array of prompt name and content pairs
     def load_predefined_prompts
-      prompts = Dir.glob(Rails.root.join('app', 'prompts', '*.md')).map do |file|
-        name = File.basename(file, '.md').titleize
+      prompts = Dir.glob(Rails.root.join("app", "prompts", "*.md")).map do |file|
+        name = File.basename(file, ".md").titleize
         content = File.read(file)
-        [name, content]
+        [ name, content ]
       end
       prompts
     end
@@ -34,8 +34,8 @@ module Leva
     # @param base_class [Class] The base class that loaded classes should inherit from
     # @return [Array<Class>] An array of loaded classes
     def load_classes_from_directory(directory, base_class)
-      classes = Dir[Rails.root.join(directory, '*.rb')].map do |file|
-        File.basename(file, '.rb').camelize.constantize
+      classes = Dir[Rails.root.join(directory, "*.rb")].map do |file|
+        File.basename(file, ".rb").camelize.constantize
       end.select { |klass| klass < base_class }
       classes.empty? ? [] : classes
     end

data/app/jobs/leva/experiment_job.rb

@@ -18,4 +18,4 @@ module Leva
       end
     end
   end
-end
+end

data/app/jobs/leva/run_eval_job.rb

@@ -37,4 +37,4 @@ module Leva
       experiment.dataset.dataset_records.count == experiment.runner_results.count
     end
   end
-end
+end

data/app/models/concerns/leva/recordable.rb

@@ -3,10 +3,10 @@ module Leva
     extend ActiveSupport::Concern
 
     included do
-      has_many :dataset_records, as: :recordable, class_name: 'Leva::DatasetRecord', dependent: :destroy
-      has_many :datasets, through: :dataset_records, class_name: 'Leva::Dataset'
-      has_many :runner_results, through: :dataset_records, class_name: 'Leva::RunnerResult'
-      has_many :evaluation_results, through: :runner_results, class_name: 'Leva::EvaluationResult'
+      has_many :dataset_records, as: :recordable, class_name: "Leva::DatasetRecord", dependent: :destroy
+      has_many :datasets, through: :dataset_records, class_name: "Leva::Dataset"
+      has_many :runner_results, through: :dataset_records, class_name: "Leva::RunnerResult"
+      has_many :evaluation_results, through: :runner_results, class_name: "Leva::EvaluationResult"
     end
 
     # @return [String] The ground truth label for the record
@@ -34,4 +34,4 @@ module Leva
       false
     end
   end
-end
+end

data/app/models/leva/dataset.rb

@@ -25,4 +25,4 @@ module Leva
       end
     end
   end
-end
+end

data/app/models/leva/evaluation_result.rb

@@ -36,4 +36,4 @@ module Leva
 
     scope :for_evaluator, ->(evaluator_class) { where(evaluator_class: evaluator_class.name) }
   end
-end
+end

data/app/models/leva/experiment.rb

@@ -40,4 +40,4 @@ module Leva
 
     serialize :evaluator_classes, coder: JSON, type: Array
   end
-end
+end

data/app/models/leva/prompt.rb

@@ -27,4 +27,4 @@ module Leva
       self.version += 1
     end
   end
-end
+end

data/app/views/layouts/leva/application.html.erb

@@ -8,7 +8,6 @@
     <%= csp_meta_tag %>
     <script src="https://cdn.tailwindcss.com"></script>
     <script src="https://cdn.jsdelivr.net/npm/stimulus@3.2.2/dist/stimulus.umd.min.js"></script>
-    <script src="<%= asset_path 'custom/prompt_preview.js' %>"></script>
     <%= yield(:head) %>
   </head>
   <body class="bg-gray-950 text-white">

data/app/views/leva/dataset_records/index.html.erb

@@ -12,38 +12,63 @@
     </div>
   </div>
 
-  <div class="bg-gray-800 rounded-lg shadow-lg overflow-hidden">
-    <table class="min-w-full divide-y divide-gray-700">
-      <thead class="bg-gray-700">
-        <tr>
-          <% @records.first.index_attributes.keys.each do |key| %>
-            <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-300 uppercase tracking-wider">
-              <%= key.to_s.humanize %>
-            </th>
-          <% end %>
-          <th scope="col" class="relative px-6 py-3">
-            <span class="sr-only">Actions</span>
-          </th>
-        </tr>
-      </thead>
-      <tbody class="bg-gray-800 divide-y divide-gray-700">
-        <% @records.each do |record| %>
-          <tr class="hover:bg-gray-700 transition-colors duration-200">
-            <% record.index_attributes.values.each do |value| %>
-              <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-300">
-                <%= value %>
-              </td>
+  <% if @records.any? %>
+    <div class="bg-gray-800 rounded-lg shadow-lg overflow-hidden">
+      <table class="min-w-full divide-y divide-gray-700">
+        <thead class="bg-gray-700">
+          <tr>
+            <% @records.first.index_attributes.keys.each do |key| %>
+              <th scope="col" class="px-6 py-3 text-left text-xs font-medium text-gray-300 uppercase tracking-wider">
+                <%= key.to_s.humanize %>
+              </th>
             <% end %>
-            <td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
-              <%= link_to 'View', dataset_dataset_record_path(@dataset, record), class: 'text-indigo-400 hover:text-indigo-300 transition-colors duration-200' %>
-            </td>
+            <th scope="col" class="relative px-6 py-3">
+              <span class="sr-only">Actions</span>
+            </th>
           </tr>
-        <% end %>
-      </tbody>
-    </table>
-  </div>
+        </thead>
+        <tbody class="bg-gray-800 divide-y divide-gray-700">
+          <% @records.each do |record| %>
+            <tr class="hover:bg-gray-700 transition-colors duration-200">
+              <% record.index_attributes.values.each do |value| %>
+                <td class="px-6 py-4 whitespace-nowrap text-sm text-gray-300">
+                  <%= value %>
+                </td>
+              <% end %>
+              <td class="px-6 py-4 whitespace-nowrap text-right text-sm font-medium">
+                <%= link_to 'View', dataset_dataset_record_path(@dataset, record), class: 'text-indigo-400 hover:text-indigo-300 transition-colors duration-200' %>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
 
-  <div class="mt-4 text-gray-400 text-sm">
-    Total records: <%= @records.count %>
-  </div>
+    <div class="mt-4 text-gray-400 text-sm">
+      Total records: <%= @records.count %>
+    </div>
+  <% else %>
+    <div class="bg-gray-800 rounded-lg shadow-lg p-12 text-center">
+      <svg class="mx-auto h-12 w-12 text-indigo-400" fill="none" viewBox="0 0 24 24" stroke="currentColor" aria-hidden="true">
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 13h6m-3-3v6m5 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
+      </svg>
+      <h3 class="mt-2 text-xl font-medium text-indigo-300">No records yet</h3>
+      <p class="mt-1 text-gray-400">This dataset doesn't contain any records yet.</p>
+      <div class="mt-4 text-sm text-gray-500">
+        <p>To add records to this dataset, use the Rails console or create them programmatically:</p>
+        <div class="mt-2 bg-gray-900 rounded px-3 py-2 text-left font-mono text-xs text-gray-300">
+          <p>dataset = Leva::Dataset.find(<%= @dataset.id %>)</p>
+          <p>dataset.add_record(your_model_instance)</p>
+        </div>
+      </div>
+      <div class="mt-6">
+        <%= link_to dataset_path(@dataset), class: "btn btn-secondary inline-flex items-center" do %>
+          <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 mr-2" viewBox="0 0 20 20" fill="currentColor">
+            <path fill-rule="evenodd" d="M9.707 16.707a1 1 0 01-1.414 0l-6-6a1 1 0 010-1.414l6-6a1 1 0 011.414 1.414L5.414 9H17a1 1 0 110 2H5.414l4.293 4.293a1 1 0 010 1.414z" clip-rule="evenodd" />
+          </svg>
+          Back to Dataset
+        <% end %>
+      </div>
+    </div>
+  <% end %>
 </div>

data/app/views/leva/experiments/new.html.erb

@@ -1,5 +1,32 @@
 <% content_for :title, 'New Experiment' %>
 <div class="container mx-auto px-4 py-8 bg-gray-950 text-white">
   <h1 class="text-3xl font-bold text-indigo-400 mb-6">New Experiment</h1>
-  <%= render 'form', experiment: @experiment %>
+  <% if @runners.present? %>
+    <%= render 'form', experiment: @experiment %>
+  <% else %>
+    <div class="bg-gray-800 rounded-lg shadow-lg p-12 text-center">
+      <svg class="mx-auto h-12 w-12 text-yellow-400" fill="none" viewBox="0 0 24 24" stroke="currentColor" aria-hidden="true">
+        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L4.082 16.5c-.77.833.192 2.5 1.732 2.5z" />
+      </svg>
+      <h3 class="mt-2 text-xl font-medium text-yellow-300">No runners available</h3>
+      <p class="mt-1 text-gray-400">You need at least one runner to create an experiment.</p>
+      <div class="mt-6 text-left bg-gray-900 rounded-lg p-6">
+        <h4 class="text-lg font-semibold text-indigo-300 mb-3">What are runners?</h4>
+        <p class="text-gray-300 mb-4">Runners define how your model processes dataset records. They contain the execution logic that takes a dataset record and returns a prediction.</p>
+        <h4 class="text-lg font-semibold text-indigo-300 mb-3">Create your first runner:</h4>
+        <div class="bg-gray-800 rounded p-3 mb-4">
+          <code class="text-green-400">rails generate leva:runner sentiment</code>
+        </div>
+        <p class="text-gray-400 text-sm">This will create <code class="text-gray-300">app/runners/sentiment_run.rb</code> with a template to implement your model logic.</p>
+      </div>
+      <div class="mt-6">
+        <%= link_to experiments_path, class: "btn btn-secondary inline-flex items-center" do %>
+          <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5 mr-2" viewBox="0 0 20 20" fill="currentColor">
+            <path fill-rule="evenodd" d="M9.707 16.707a1 1 0 01-1.414 0l-6-6a1 1 0 010-1.414l6-6a1 1 0 011.414 1.414L5.414 9H17a1 1 0 110 2H5.414l4.293 4.293a1 1 0 010 1.414z" clip-rule="evenodd" />
+          </svg>
+          Back to Experiments
+        <% end %>
+      </div>
+    </div>
+  <% end %>
 </div>

data/app/views/leva/workbench/_prompt_content.html.erb

@@ -48,19 +48,65 @@
           </svg>
         </h2>
         <div class="bg-gray-800 p-3 rounded-lg text-sm hidden" data-collapsible-target="content">
-          <% @dataset_record.recordable.to_llm_context.each do |key, value| %>
-            <details class="mb-2">
-              <summary class="text-green-400 cursor-pointer flex items-center justify-between">
-                <span>{{ <%= key %> }}</span>
-                <button class="btn btn-small text-blue-400 hover:text-blue-300 flex items-center" data-action="clipboard#copy" data-clipboard-source="liquidTag<%= key %>">
-                  <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 mr-1" fill="none" viewBox="0 0 24 24" stroke="currentColor">
-                    <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
-                  </svg>
-                  Copy
-                </button>
-              </summary>
-              <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap break-words overflow-x-auto max-w-full" id="liquidTag<%= key %>"><%= value.to_s %></pre>
-            </details>
+          <% # Use merged context if available, otherwise just record context %>
+          <% context_to_display = @merged_context || @dataset_record.recordable.to_llm_context %>
+          
+          <% if @record_context && @runner_context %>
+            <!-- Record Context -->
+            <div class="mb-4">
+              <h3 class="text-xs font-semibold text-gray-400 mb-2">FROM RECORD:</h3>
+              <% @record_context.each do |key, value| %>
+                <details class="mb-2">
+                  <summary class="text-green-400 cursor-pointer flex items-center justify-between">
+                    <span>{{ <%= key %> }}</span>
+                    <button class="btn btn-small text-blue-400 hover:text-blue-300 flex items-center" data-action="clipboard#copy" data-clipboard-source="liquidTag<%= key %>">
+                      <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 mr-1" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                        <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
+                      </svg>
+                      Copy
+                    </button>
+                  </summary>
+                  <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap break-words overflow-x-auto max-w-full" id="liquidTag<%= key %>"><%= value.to_s %></pre>
+                </details>
+              <% end %>
+            </div>
+            
+            <!-- Runner Context -->
+            <% if @runner_context.any? %>
+              <div class="mb-4">
+                <h3 class="text-xs font-semibold text-gray-400 mb-2">FROM RUNNER:</h3>
+                <% @runner_context.each do |key, value| %>
+                  <details class="mb-2">
+                    <summary class="text-yellow-400 cursor-pointer flex items-center justify-between">
+                      <span>{{ <%= key %> }}</span>
+                      <button class="btn btn-small text-blue-400 hover:text-blue-300 flex items-center" data-action="clipboard#copy" data-clipboard-source="liquidTag<%= key %>">
+                        <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 mr-1" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                          <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
+                        </svg>
+                        Copy
+                      </button>
+                    </summary>
+                    <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap break-words overflow-x-auto max-w-full" id="liquidTag<%= key %>"><%= value.to_s %></pre>
+                  </details>
+                <% end %>
+              </div>
+            <% end %>
+          <% else %>
+            <!-- Fallback to original display -->
+            <% context_to_display.each do |key, value| %>
+              <details class="mb-2">
+                <summary class="text-green-400 cursor-pointer flex items-center justify-between">
+                  <span>{{ <%= key %> }}</span>
+                  <button class="btn btn-small text-blue-400 hover:text-blue-300 flex items-center" data-action="clipboard#copy" data-clipboard-source="liquidTag<%= key %>">
+                    <svg xmlns="http://www.w3.org/2000/svg" class="h-4 w-4 mr-1" fill="none" viewBox="0 0 24 24" stroke="currentColor">
+                      <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
+                    </svg>
+                    Copy
+                  </button>
+                </summary>
+                <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap break-words overflow-x-auto max-w-full" id="liquidTag<%= key %>"><%= value.to_s %></pre>
+              </details>
+            <% end %>
           <% end %>
         </div>
       </div>
@@ -77,7 +123,7 @@
             Copy
           </button>
         </div>
-        <pre class="w-full bg-gray-800 text-white p-3 rounded-lg text-sm whitespace-pre-wrap overflow-x-auto break-words max-w-full" id="fullPrompt"><%= Liquid::Template.parse(@selected_prompt.user_prompt).render(@dataset_record.recordable.to_llm_context.stringify_keys) %></pre>
+        <pre class="w-full bg-gray-800 text-white p-3 rounded-lg text-sm whitespace-pre-wrap overflow-x-auto break-words max-w-full" id="fullPrompt"><%= Liquid::Template.parse(@selected_prompt.user_prompt).render((@merged_context || @dataset_record.recordable.to_llm_context).stringify_keys) %></pre>
       </div>
     <% end %>
     <div class="text-sm text-center" data-prompt-autosave-target="status"></div>

data/config/routes.rb

@@ -1,21 +1,21 @@
 Leva::Engine.routes.draw do
-  root 'workbench#index'
+  root "workbench#index"
 
   resources :datasets do
-    resources :dataset_records, path: 'records', only: [:index, :show]
+    resources :dataset_records, path: "records", only: [ :index, :show ]
   end
-  resources :experiments, except: [:destroy] do
+  resources :experiments, except: [ :destroy ] do
     member do
       post :rerun
     end
-    resources :runner_results, only: [:show]
+    resources :runner_results, only: [ :show ]
   end
   resources :prompts
-  resources :workbench, only: [:index, :new, :create, :edit, :update] do
+  resources :workbench, only: [ :index, :new, :create, :edit, :update ] do
     collection do
-      post 'run'
-      post 'run_all_evals'
-      post 'run_evaluator'
+      post "run"
+      post "run_all_evals"
+      post "run_evaluator"
     end
   end
-end
+end

data/db/migrate/20240813173033_create_leva_dataset_records.rb

@@ -3,6 +3,7 @@ class CreateLevaDatasetRecords < ActiveRecord::Migration[7.2]
     create_table :leva_dataset_records do |t|
       t.references :dataset, null: false, foreign_key: { to_table: :leva_datasets }
       t.references :recordable, polymorphic: true, null: false
+      t.text :actual_result
 
       t.timestamps
     end

data/db/migrate/20240813173035_create_leva_experiments.rb

@@ -7,6 +7,8 @@ class CreateLevaExperiments < ActiveRecord::Migration[7.2]
       t.references :prompt, null: true, foreign_key: { to_table: :leva_prompts }
       t.integer :status
       t.text :metadata
+      t.string :runner_class
+      t.text :evaluator_classes
 
       t.timestamps
     end

data/db/migrate/{20240816201419_create_leva_runner_results.rb → 20240813173040_create_leva_runner_results.rb}

@@ -1,9 +1,12 @@
 class CreateLevaRunnerResults < ActiveRecord::Migration[7.2]
   def change
     create_table :leva_runner_results do |t|
-      t.references :experiment, null: false, foreign_key: { to_table: :leva_experiments }
+      t.references :experiment, null: true, foreign_key: { to_table: :leva_experiments }
       t.references :dataset_record, null: false, foreign_key: { to_table: :leva_dataset_records }
+      t.references :prompt, null: false, foreign_key: { to_table: :leva_prompts }
+      t.integer :prompt_version
       t.text :prediction
+      t.string :runner_class
 
       t.timestamps
     end

data/db/migrate/20240813173050_create_leva_evaluation_results.rb

@@ -1,11 +1,11 @@
 class CreateLevaEvaluationResults < ActiveRecord::Migration[7.2]
   def change
     create_table :leva_evaluation_results do |t|
-      t.references :experiment, null: false, foreign_key: { to_table: :leva_experiments }
+      t.references :experiment, null: true, foreign_key: { to_table: :leva_experiments }
       t.references :dataset_record, null: false, foreign_key: { to_table: :leva_dataset_records }
-      t.string :prediction
+      t.references :runner_result, null: false, foreign_key: { to_table: :leva_runner_results }
+      t.string :evaluator_class, null: false
       t.float :score
-      t.string :label
 
       t.timestamps
     end

data/lib/generators/leva/eval_generator.rb

@@ -3,10 +3,10 @@
 module Leva
   module Generators
     class EvalGenerator < Rails::Generators::NamedBase
-      source_root File.expand_path('templates', __dir__)
+      source_root File.expand_path("templates", __dir__)
 
       def create_eval_file
-        template 'eval.rb.erb', File.join('app/evals', class_path, "#{file_name}_eval.rb")
+        template "eval.rb.erb", File.join("app/evals", class_path, "#{file_name}_eval.rb")
       end
 
       private
@@ -16,8 +16,8 @@ module Leva
       end
 
       def remove_possible_suffix(name)
-        name.sub(/_?eval$/i, '')
+        name.sub(/_?eval$/i, "")
       end
     end
   end
-end
+end

data/lib/generators/leva/runner_generator.rb

@@ -3,10 +3,10 @@
 module Leva
   module Generators
     class RunnerGenerator < Rails::Generators::NamedBase
-      source_root File.expand_path('templates', __dir__)
+      source_root File.expand_path("templates", __dir__)
 
       def create_runner_file
-        template 'runner.rb.erb', File.join('app/runners', class_path, "#{file_name}_run.rb")
+        template "runner.rb.erb", File.join("app/runners", class_path, "#{file_name}_run.rb")
       end
 
       private
@@ -16,8 +16,8 @@ module Leva
       end
 
       def remove_possible_suffix(name)
-        name.sub(/_?runner$/i, '')
+        name.sub(/_?runner$/i, "")
       end
     end
   end
-end
+end

data/lib/generators/leva/templates/runner.rb.erb

@@ -33,4 +33,24 @@ class <%= class_name %>Run < Leva::BaseRun
   #   # For example:
   #   # /\<result\>(.*?)\<\/result\>/
   # end
+
+  # Override this method to provide additional context for LLM prompts
+  # This is useful for expensive operations or runner-specific data
+  # that you don't want to compute in the record's to_llm_context
+  # 
+  # @param record [YourRecordClass] The record to generate context for
+  # @return [Hash] Additional context to merge with the record's context
+  # def to_llm_context(record)
+  #   # Example: Add runner-specific expensive computations
+  #   {
+  #     # Count of similar records (expensive database query)
+  #     similar_<%= singular_name %>_count: record.class.where(
+  #       "some_field LIKE ?", "%#{record.some_field.split.first}%"
+  #     ).count,
+  #     
+  #     # Runner-specific configuration
+  #     model_version: "v1.2.3",
+  #     temperature: 0.7
+  #   }
+  # end
 end

data/lib/leva/version.rb

@@ -1,3 +1,3 @@
 module Leva
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
 end

data/lib/leva.rb

@@ -65,6 +65,7 @@ module Leva
       # Expose these to the subclass execution
       @experiment = experiment
       @prompt = prompt
+      @dataset_record = dataset_record
 
       result = execute(dataset_record.recordable)
       RunnerResult.create!(
@@ -76,13 +77,25 @@ module Leva
       )
     end
 
+    # Gets the merged LLM context for the current execution.
+    # Combines the record's context with the runner's additional context.
+    #
+    # @return [Hash] The merged context for LLM prompt rendering
+    def merged_llm_context
+      return {} unless @dataset_record
+
+      record_context = @dataset_record.recordable.to_llm_context
+      runner_context = to_llm_context(@dataset_record.recordable)
+      record_context.merge(runner_context)
+    end
+
     # @param runner_result [Leva::RunnerResult] The runner result to parse
     # @return [Array<String>] The parsed predictions
     def parsed_predictions(runner_result)
       if extract_regex_pattern(runner_result)
         runner_result.prediction.scan(extract_regex_pattern(runner_result)).map { |match| match.first&.strip }.compact
       else
-        [runner_result.prediction]
+        [ runner_result.prediction ]
       end
     end
 
@@ -97,6 +110,15 @@ module Leva
     def ground_truth(runner_result)
       runner_result.dataset_record.ground_truth
     end
+
+    # Provides additional LLM context specific to this runner.
+    # Override in subclasses to add expensive or runner-specific context.
+    #
+    # @param record [Object] The recordable object to generate context for
+    # @return [Hash] Additional context to merge with the record's context
+    def to_llm_context(record)
+      {}
+    end
   end
 
   # Base class for all evaluation implementations in Leva.
@@ -134,4 +156,4 @@ module Leva
       )
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: leva
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Kieran Klaassen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-25 00:00:00.000000000 Z
+date: 2025-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -103,15 +103,8 @@ files:
 - db/migrate/20240813173033_create_leva_dataset_records.rb
 - db/migrate/20240813173034_create_leva_prompts.rb
 - db/migrate/20240813173035_create_leva_experiments.rb
+- db/migrate/20240813173040_create_leva_runner_results.rb
 - db/migrate/20240813173050_create_leva_evaluation_results.rb
-- db/migrate/20240816201419_create_leva_runner_results.rb
-- db/migrate/20240816201433_update_leva_evaluation_results.rb
-- db/migrate/20240821163608_make_experiment_optional_for_runner_results.rb
-- db/migrate/20240821181934_add_prompt_to_leva_runner_results.rb
-- db/migrate/20240821183153_add_runner_and_evaluator_to_leva_experiments.rb
-- db/migrate/20240821191713_add_actual_result_to_leva_dataset_records.rb
-- db/migrate/20240822143201_remove_actual_result_from_leva_runner_results.rb
-- db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb
 - lib/generators/leva/eval_generator.rb
 - lib/generators/leva/runner_generator.rb
 - lib/generators/leva/templates/eval.rb.erb
@@ -119,7 +112,6 @@ files:
 - lib/leva.rb
 - lib/leva/engine.rb
 - lib/leva/version.rb
-- lib/tasks/auto_annotate_models.rake
 - lib/tasks/leva_tasks.rake
 homepage: https://github.com/kieranklaassen/leva
 licenses:

data/db/migrate/20240816201433_update_leva_evaluation_results.rb

@@ -1,8 +0,0 @@
-class UpdateLevaEvaluationResults < ActiveRecord::Migration[7.2]
-  def change
-    add_reference :leva_evaluation_results, :runner_result, null: false, foreign_key: { to_table: :leva_runner_results }
-    add_column :leva_evaluation_results, :evaluator_class, :string, null: false
-    remove_column :leva_evaluation_results, :prediction, :string
-    remove_column :leva_evaluation_results, :label, :string
-  end
-end

data/db/migrate/20240821163608_make_experiment_optional_for_runner_results.rb

@@ -1,6 +0,0 @@
-class MakeExperimentOptionalForRunnerResults < ActiveRecord::Migration[7.2]
-  def change
-    change_column_null :leva_runner_results, :experiment_id, true
-    change_column_null :leva_evaluation_results, :experiment_id, true
-  end
-end

data/db/migrate/20240821181934_add_prompt_to_leva_runner_results.rb

@@ -1,5 +0,0 @@
-class AddPromptToLevaRunnerResults < ActiveRecord::Migration[7.2]
-  def change
-    add_reference :leva_runner_results, :prompt, null: false, foreign_key: { to_table: :leva_prompts }
-  end
-end

data/db/migrate/20240821183153_add_runner_and_evaluator_to_leva_experiments.rb

@@ -1,6 +0,0 @@
-class AddRunnerAndEvaluatorToLevaExperiments < ActiveRecord::Migration[7.2]
-  def change
-    add_column :leva_experiments, :runner_class, :string
-    add_column :leva_experiments, :evaluator_classes, :text
-  end
-end

data/db/migrate/20240821191713_add_actual_result_to_leva_dataset_records.rb

@@ -1,5 +0,0 @@
-class AddActualResultToLevaDatasetRecords < ActiveRecord::Migration[7.2]
-  def change
-    add_column :leva_runner_results, :actual_result, :text
-  end
-end

data/db/migrate/20240822143201_remove_actual_result_from_leva_runner_results.rb

@@ -1,5 +0,0 @@
-class RemoveActualResultFromLevaRunnerResults < ActiveRecord::Migration[7.2]
-  def change
-    remove_column :leva_runner_results, :actual_result, :text
-  end
-end

data/db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb

@@ -1,5 +0,0 @@
-class AddRunnerClassToLevaRunnerResults < ActiveRecord::Migration[7.2]
-  def change
-    add_column :leva_runner_results, :runner_class, :string
-  end
-end

data/lib/tasks/auto_annotate_models.rake

@@ -1,59 +0,0 @@
-# NOTE: only doing this in development as some production environments (Heroku)
-# NOTE: are sensitive to local FS writes, and besides -- it's just not proper
-# NOTE: to have a dev-mode tool do its thing in production.
-if Rails.env.development?
-  require 'annotate'
-  task :set_annotation_options do
-    # You can override any of these by setting an environment variable of the
-    # same name.
-    Annotate.set_defaults(
-      'active_admin'                => 'false',
-      'additional_file_patterns'    => [],
-      'routes'                      => 'false',
-      'models'                      => 'true',
-      'position_in_routes'          => 'before',
-      'position_in_class'           => 'before',
-      'position_in_test'            => 'before',
-      'position_in_fixture'         => 'before',
-      'position_in_factory'         => 'before',
-      'position_in_serializer'      => 'before',
-      'show_foreign_keys'           => 'true',
-      'show_complete_foreign_keys'  => 'false',
-      'show_indexes'                => 'true',
-      'simple_indexes'              => 'false',
-      'model_dir'                   => 'app/models',
-      'root_dir'                    => '',
-      'include_version'             => 'false',
-      'require'                     => '',
-      'exclude_tests'               => 'false',
-      'exclude_fixtures'            => 'false',
-      'exclude_factories'           => 'false',
-      'exclude_serializers'         => 'false',
-      'exclude_scaffolds'           => 'true',
-      'exclude_controllers'         => 'true',
-      'exclude_helpers'             => 'true',
-      'exclude_sti_subclasses'      => 'false',
-      'ignore_model_sub_dir'        => 'false',
-      'ignore_columns'              => nil,
-      'ignore_routes'               => nil,
-      'ignore_unknown_models'       => 'false',
-      'hide_limit_column_types'     => 'integer,bigint,boolean',
-      'hide_default_column_types'   => 'json,jsonb,hstore',
-      'skip_on_db_migrate'          => 'false',
-      'format_bare'                 => 'true',
-      'format_rdoc'                 => 'false',
-      'format_yard'                 => 'false',
-      'format_markdown'             => 'false',
-      'sort'                        => 'false',
-      'force'                       => 'false',
-      'frozen'                      => 'false',
-      'classified_sort'             => 'true',
-      'trace'                       => 'false',
-      'wrapper_open'                => nil,
-      'wrapper_close'               => nil,
-      'with_comment'                => 'true'
-    )
-  end
-
-  Annotate.load_tasks
-end