leva 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10570fd9a6cdbf979cf9538963033a752ef0cb7d4d1966e9a002408f875a5aa2
-  data.tar.gz: e194a81e5e53ddf22b3aee902aab09191ccf39836385c885692f95b11fbcdd46
+  metadata.gz: 7a1b17552bf84bd7721ee48cd83caf82f199d2b79218296f153015e144211356
+  data.tar.gz: eab228d96d36c3afce52a403d728ff93f6729730cf7cbef35411958443b6327c
 SHA512:
-  metadata.gz: 7b534a0aceff9d67f1a0f00d3b70993fec440633ef20005adf5ac866772bd57899fb7412073a7deaa2a7dfac856def72bc51a94ed6a5eb26a21818ec405fddfc
-  data.tar.gz: 42551d07b819337c1ac780aa542fb41af2218e257c9f1bf5d11d21dd351e64f5c6942d2354cbce2dead1b0232e62140fe16a1c5a308c6ba2b9fe8659e3b5b88b
+  metadata.gz: 0e2a98bde71873044ceca0be67213b2fe4d0547eb38430ea142f3425e57c6b1b14adad5644d6c747662e21587e29ca65a9205c2a85b2f14ca67752eda87fa607
+  data.tar.gz: dedfbe490f7ded9daf6af1d43e0c5611c4020beb5a9150d949e31ed4b32d56f358c4a15bde9e43b96c64ee9bffd50819a47343d0181bc415c1e3d04e20083610

data/README.md

@@ -1,6 +1,6 @@
 # Leva - Flexible Evaluation Framework for Language Models
 
-Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic.
+Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets on production models. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic on production data with security in mind.
 
 ## Installation
 
@@ -20,120 +20,119 @@ $ bundle install
 
 ### 1. Setting up Datasets
 
-First, create a dataset and add any ActiveRecord records:
+First, create a dataset and add any ActiveRecord records you want to evaluate against:
 
 ```ruby
-dataset = Dataset.create(name: "Sentiment Analysis Dataset")
-
-dataset.records << TextContent.create(text: "I love this product!", expected_label: "Positive")
-dataset.records << TextContent.create(text: "Terrible experience", expected_label: "Negative")
-dataset.records << TextContent.create(text: "I's ok", expected_label: "Neutral")
+dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset")
+dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive")
+dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative")
+dataset.add_record TextContent.create(text: "I's ok", expected_label: "Neutral")
 ```
 
-> In this case the TextContent model is the ActiveRecord model from your own application.
-
-### 2. Implementing Evals
+### 2. Implementing Runs
 
-Create evals by adding new files in `app/evals/`. Each eval implements both the evaluation logic and how to run it. Here are some examples:
+Create a run class to handle the execution of your inference logic:
 
 ```bash
-$ rails generate leva:eval Sentiment
+rails generate leva:runner sentiment
 ```
 
-#### Sentiment Evaluation (app/evals/sentiment_eval.rb)
-
 ```ruby
-class SentimentEval < Leva::BaseEval
-  leva_dataset_record_class "TextContent"
-
-  def run_each(record)
-    prediction = label_sentiment(record.text)
-    score = calculate_score(prediction, record.expected_label)
-
-    Leva::Result.new(
-      label: 'sentiment',
-      prediction: prediction,
-      score: score
-    )
+class SentimentRun < Leva::BaseRun
+  def execute(record)
+    # Your model execution logic here
+    # This could involve calling an API, running a local model, etc.
+    # Return the model's output
   end
+end
+```
+
+### 3. Implementing Evals
+
+Create one or more eval classes to evaluate the model's output:
+
+```bash
+rails generate leva:eval sentiment_accuracy
+```
 
-  private
-
-  def label_sentiment(text)
-    # Simple sentiment analysis logic, use LLM to label the sentiment yourself
-    text = text.downcase
-    if text.include?('love')
-      'Positive'
-    elsif text.include?('terrible')
-      'Negative'
-    else
-      'Neutral'
-    end
+```ruby
+class SentimentAccuracyEval < Leva::BaseEval
+  def evaluate(prediction, expected)
+    score = prediction == expected ? 1.0 : 0.0
+    Leva::Result.new(label: 'sentiment_accuracy', score: score)
   end
+end
 
-  def calculate_score(prediction, expected)
-    prediction == expected ? 1.0 : 0.0
+class SentimentF1Eval < Leva::BaseEval
+  def evaluate(prediction, expected)
+    # Calculate F1 score
+    # ...
+    Leva::Result.new(label: 'sentiment_f1', score: f1_score)
   end
 end
 ```
 
-### 3. Running Experiments
+### 4. Running Experiments
 
-You can run experiments with different evals:
+You can run experiments with different runs and evals:
 
 ```ruby
-sentiment_experiment = Experiment.create!(name: "Sentiment Analysis", dataset: dataset)
-SentimentEval.run_experiment(sentiment_experiment)
+experiment = Leva::Experiment.create!(name: "Sentiment Analysis", dataset: dataset)
+
+run = SentimentRun.new
+evals = [SentimentAccuracyEval.new, SentimentF1Eval.new]
+
+Leva.run_evaluation(experiment: experiment, run: run, evals: evals)
 ```
 
-You can also run an experiment with a prompt so you can use a LLM to evaluate the dataset:
+### 5. Using Prompts
+
+You can also use prompts with your runs:
 
 ```ruby
 prompt = Leva::Prompt.create!(
   name: "Sentiment Analysis",
   version: 1,
   system_prompt: "You are an expert at analyzing text and returning the sentiment.",
-  user_prompt: "Please analyze the following text and return the sentiment as Positive, Negative, or Neutral.  \n\n {{TEXT}}",
-  metadata: {
-    model: "gpt-4o",
-    temperature: 0.5
-  }
+  user_prompt: "Please analyze the following text and return the sentiment as Positive, Negative, or Neutral.\n\n{{TEXT}}",
+  metadata: { model: "gpt-4", temperature: 0.5 }
 )
 
-sentiment_experiment = Experiment.create!(
+experiment = Leva::Experiment.create!(
   name: "Sentiment Analysis with LLM",
   dataset: dataset,
   prompt: prompt
 )
 
-SentimentEval.run_experiment(sentiment_experiment)
+run = SentimentRun.new
+evals = [SentimentAccuracyEval.new, SentimentF1Eval.new]
+
+Leva.run_evaluation(experiment: experiment, run: run, evals: evals)
 ```
 
-### 4. Analyzing Results
+### 6. Analyzing Results
 
 After the experiments are complete, analyze the results:
 
 ```ruby
-
-results = experiment.evaluation_results
-average_score = results.average(:score)
-count = results.count
-
-puts "Experiment: #{experiment.name}"
-puts "Average Score: #{average_score}"
-puts "Number of Evaluations: #{count}"
+experiment.evaluation_results.group_by(&:label).each do |label, results|
+  average_score = results.average(&:score)
+  puts "#{label.capitalize} Average Score: #{average_score}"
+end
 ```
 
 ## Configuration
 
-If your evals require API keys or other configurations, ensure you set these up in your Rails credentials or environment variables.
+Ensure you set up any required API keys or other configurations in your Rails credentials or environment variables.
 
 ## Leva's Components
 
 ### Classes
 
-- `Leva::BaseEval`: The base class for all evals. Override the `run` method in your eval classes.
-- `Leva::Result`: The result of an evaluation.
+- `Leva`: Handles the process of running experiments.
+- `Leva::BaseRun`: Base class for run implementations.
+- `Leva::BaseEval`: Base class for eval implementations.
+- `Leva::Result`: Represents the result of an evaluation.
 
 ### Models
 

data/app/controllers/leva/application_controller.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module Leva
   class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
   end
-end
+end

data/app/controllers/leva/datasets_controller.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Leva
+  class DatasetsController < ApplicationController
+    before_action :set_dataset, only: [:show, :edit, :update, :destroy]
+
+    # GET /datasets
+    # @return [void]
+    def index
+      @datasets = Dataset.all
+    end
+
+    # GET /datasets/1
+    # @return [void]
+    def show
+    end
+
+    # GET /datasets/new
+    # @return [void]
+    def new
+      @dataset = Dataset.new
+    end
+
+    # GET /datasets/1/edit
+    # @return [void]
+    def edit
+    end
+
+    # POST /datasets
+    # @return [void]
+    def create
+      @dataset = Dataset.new(dataset_params)
+
+      if @dataset.save
+        redirect_to @dataset, notice: 'Dataset was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    # PATCH/PUT /datasets/1
+    # @return [void]
+    def update
+      if @dataset.update(dataset_params)
+        redirect_to @dataset, notice: 'Dataset was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    # DELETE /datasets/1
+    # @return [void]
+    def destroy
+      @dataset.destroy
+      redirect_to datasets_url, notice: 'Dataset was successfully destroyed.'
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    # @return [void]
+    def set_dataset
+      @dataset = Dataset.find(params[:id])
+    end
+
+    # Only allow a list of trusted parameters through.
+    # @return [ActionController::Parameters]
+    def dataset_params
+      params.require(:dataset).permit(:name, :description)
+    end
+  end
+end

data/app/controllers/leva/experiments_controller.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Leva
+  class ExperimentsController < ApplicationController
+    before_action :set_experiment, only: [:show, :edit, :update, :destroy]
+
+    # GET /experiments
+    # @return [void]
+    def index
+      @experiments = Experiment.all
+    end
+
+    # GET /experiments/1
+    # @return [void]
+    def show
+    end
+
+    # GET /experiments/new
+    # @return [void]
+    def new
+      @experiment = Experiment.new
+    end
+
+    # GET /experiments/1/edit
+    # @return [void]
+    def edit
+    end
+
+    # POST /experiments
+    # @return [void]
+    def create
+      @experiment = Experiment.new(experiment_params)
+
+      if @experiment.save
+        ExperimentJob.perform_later(@experiment)
+        redirect_to @experiment, notice: 'Experiment was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    # PATCH/PUT /experiments/1
+    # @return [void]
+    def update
+      if @experiment.update(experiment_params)
+        redirect_to @experiment, notice: 'Experiment was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    # DELETE /experiments/1
+    # @return [void]
+    def destroy
+      @experiment.destroy
+      redirect_to experiments_url, notice: 'Experiment was successfully destroyed.'
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    # @return [void]
+    def set_experiment
+      @experiment = Experiment.find(params[:id])
+    end
+
+    # Only allow a list of trusted parameters through.
+    # @return [ActionController::Parameters]
+    def experiment_params
+      params.require(:experiment).permit(:name, :description, :dataset_id)
+    end
+  end
+end

data/app/controllers/leva/workbench_controller.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Leva
+  class WorkbenchController < ApplicationController
+    # GET /workbench
+    # @return [void]
+    def index
+      @prompts = Prompt.all
+      @selected_prompt = Prompt.first || Prompt.create!(name: "Test Prompt", version: 1, system_prompt: "You are a helpful assistant.", user_prompt: "Hello, how can I help you today?")
+      @evaluators = ['Evaluator 1', 'Evaluator 2', 'Evaluator 3']
+    end
+
+    # GET /workbench/new
+    # @return [void]
+    def new
+      @experiment = Experiment.new
+    end
+
+    # GET /workbench/1
+    # @return [void]
+    def show
+      @experiment = Experiment.find(params[:id])
+    end
+
+    def run
+      # Implement the logic for running the prompt
+      redirect_to workbench_index_path, notice: 'Prompt run successfully'
+    end
+
+    def run_with_evaluation
+      # Implement the logic for running the prompt with evaluation
+      redirect_to workbench_index_path, notice: 'Prompt run with evaluation successfully'
+    end
+
+    def run_evaluator
+      # Implement the logic for running a single evaluator
+      redirect_to workbench_index_path, notice: 'Evaluator run successfully'
+    end
+  end
+end

data/app/evals/test_sentiment_accuracy_eval.rb

@@ -0,0 +1,6 @@
+class TestSentimentAccuracyEval < Leva::BaseEval
+  def evaluate(prediction, expected)
+    score = prediction == expected ? 1.0 : 0.0
+    Leva::Result.new(label: 'sentiment_accuracy', score: score)
+  end
+end

data/app/jobs/leva/experiment_job.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Leva
+  class ExperimentJob < ApplicationJob
+    queue_as :default
+
+    # Perform the experiment
+    #
+    # @param experiment [Experiment] The experiment to run
+    # @return [void]
+    def perform(eval, record)
+      result = eval.run_each(record)
+      eval.save_result(result)
+    end
+  end
+end

data/app/models/leva/dataset.rb

@@ -10,7 +10,10 @@
 module Leva
   class Dataset < ApplicationRecord
     has_many :dataset_records, dependent: :destroy
-    has_many :records, through: :dataset_records, source: :recordable
     has_many :experiments, dependent: :destroy
+
+    def add_record(record)
+      dataset_records.create(recordable: record)
+    end
   end
 end

data/app/models/leva/dataset_record.rb

@@ -16,7 +16,7 @@
 #
 # Foreign Keys
 #
-#  dataset_id  (dataset_id => datasets.id)
+#  dataset_id  (dataset_id => leva_datasets.id)
 #
 module Leva
   class DatasetRecord < ApplicationRecord

data/app/models/leva/evaluation_result.rb

@@ -18,12 +18,14 @@
 #
 # Foreign Keys
 #
-#  dataset_record_id  (dataset_record_id => dataset_records.id)
-#  experiment_id      (experiment_id => experiments.id)
+#  dataset_record_id  (dataset_record_id => leva_dataset_records.id)
+#  experiment_id      (experiment_id => leva_experiments.id)
 #
 module Leva
   class EvaluationResult < ApplicationRecord
     belongs_to :experiment
     belongs_to :dataset_record
+
+    delegate :record, to: :dataset_record, allow_nil: true
   end
 end

data/app/models/leva/experiment.rb

@@ -9,7 +9,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  dataset_id :integer          not null
-#  prompt_id  :integer          not null
+#  prompt_id  :integer
 #
 # Indexes
 #
@@ -18,12 +18,14 @@
 #
 # Foreign Keys
 #
-#  dataset_id  (dataset_id => datasets.id)
-#  prompt_id   (prompt_id => prompts.id)
+#  dataset_id  (dataset_id => leva_datasets.id)
+#  prompt_id   (prompt_id => leva_prompts.id)
 #
 module Leva
   class Experiment < ApplicationRecord
     belongs_to :dataset
-    belongs_to :prompt
+    belongs_to :prompt, optional: true
+
+    has_many :evaluation_results, dependent: :destroy
   end
 end

data/app/runners/test_sentiment_run.rb

@@ -0,0 +1,13 @@
+class TestSentimentRun < Leva::BaseRun
+  def execute(record)
+    # Simple sentiment analysis logic for testing
+    case record.content.downcase
+    when /love|great|excellent/
+      "Positive"
+    when /terrible|bad|awful/
+      "Negative"
+    else
+      "Neutral"
+    end
+  end
+end

data/app/views/layouts/leva/application.html.erb

@@ -1,17 +1,30 @@
 <!DOCTYPE html>
-<html>
-<head>
-  <title>Leva</title>
-  <%= csrf_meta_tags %>
-  <%= csp_meta_tag %>
-
-  <%= yield :head %>
-
-  <%= stylesheet_link_tag    "leva/application", media: "all" %>
-</head>
-<body>
-
-<%= yield %>
-
-</body>
-</html>
+<html lang="en" class="bg-gray-900">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Leva - <%= yield(:title) || 'AI Evaluation Engine' %></title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+    <script src="https://cdn.tailwindcss.com"></script>
+  </head>
+  <body class="bg-gray-900 text-white">
+    <nav class="bg-gray-800 border-b border-gray-700">
+      <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
+        <div class="flex justify-between h-16">
+          <div class="flex">
+            <%= link_to 'Leva', leva.root_path, class: 'flex-shrink-0 flex items-center text-2xl font-bold text-indigo-400' %>
+            <div class="hidden sm:ml-6 sm:flex sm:space-x-8">
+              <%= link_to 'Workbench', leva.workbench_index_path, class: "border-transparent #{request.path.start_with?(leva.workbench_index_path) ? 'border-indigo-500 text-white' : 'text-gray-400 hover:border-gray-700 hover:text-gray-300'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>
+              <%= link_to 'Datasets', leva.datasets_path, class: "border-transparent #{request.path.start_with?(leva.datasets_path) ? 'border-indigo-500 text-white' : 'text-gray-400 hover:border-gray-700 hover:text-gray-300'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>
+              <%= link_to 'Experiments', leva.experiments_path, class: "border-transparent #{request.path.start_with?(leva.experiments_path) ? 'border-indigo-500 text-white' : 'text-gray-400 hover:border-gray-700 hover:text-gray-300'} inline-flex items-center px-1 pt-1 border-b-2 text-sm font-medium" %>
+            </div>
+          </div>
+        </div>
+      </div>
+    </nav>
+    <main class="bg-gray-900">
+      <%= yield %>
+    </main>
+  </body>
+</html>

data/app/views/leva/datasets/index.html.erb

@@ -0,0 +1,43 @@
+<% content_for :title, 'Datasets' %>
+<div class="px-4 sm:px-6 lg:px-8">
+  <div class="sm:flex sm:items-center">
+    <div class="sm:flex-auto">
+      <h1 class="text-2xl font-semibold text-gray-900">Datasets</h1>
+      <p class="mt-2 text-sm text-gray-700">A list of all datasets in your account.</p>
+    </div>
+    <div class="mt-4 sm:mt-0 sm:ml-16 sm:flex-none">
+      <%= link_to 'Add Dataset', new_dataset_path, class: 'btn btn-primary' %>
+    </div>
+  </div>
+  <div class="mt-8 flex flex-col">
+    <div class="-my-2 -mx-4 overflow-x-auto sm:-mx-6 lg:-mx-8">
+      <div class="inline-block min-w-full py-2 align-middle md:px-6 lg:px-8">
+        <div class="overflow-hidden shadow ring-1 ring-black ring-opacity-5 md:rounded-lg">
+          <table class="min-w-full divide-y divide-gray-300">
+            <thead class="bg-gray-50">
+              <tr>
+                <th scope="col" class="py-3.5 pl-4 pr-3 text-left text-sm font-semibold text-gray-900 sm:pl-6">Name</th>
+                <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">Description</th>
+                <th scope="col" class="relative py-3.5 pl-3 pr-4 sm:pr-6">
+                  <span class="sr-only">Actions</span>
+                </th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-gray-200 bg-white">
+              <% @datasets.each do |dataset| %>
+                <tr>
+                  <td class="whitespace-nowrap py-4 pl-4 pr-3 text-sm font-medium text-gray-900 sm:pl-6"><%= dataset.name %></td>
+                  <td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500"><%= dataset.description %></td>
+                  <td class="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-6">
+                    <%= link_to 'View', dataset_path(dataset), class: 'text-indigo-600 hover:text-indigo-900' %>
+                    <%= link_to 'Edit', edit_dataset_path(dataset), class: 'ml-4 text-indigo-600 hover:text-indigo-900' %>
+                  </td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/leva/datasets/show.html.erb

@@ -0,0 +1,13 @@
+<% content_for :title, @dataset.name %>
+<div class="px-4 sm:px-6 lg:px-8">
+  <div class="sm:flex sm:items-center">
+    <div class="sm:flex-auto">
+      <h1 class="text-2xl font-semibold text-gray-900"><%= @dataset.name %></h1>
+      <p class="mt-2 text-sm text-gray-700"><%= @dataset.description %></p>
+    </div>
+    <div class="mt-4 sm:mt-0 sm:ml-16 sm:flex-none">
+      <%= link_to 'Edit Dataset', edit_dataset_path(@dataset), class: 'btn btn-primary' %>
+    </div>
+  </div>
+  <!-- Add more dataset details here -->
+</div>

data/app/views/leva/experiments/index.html.erb

@@ -0,0 +1,44 @@
+<% content_for :title, 'Experiments' %>
+<div class="px-4 sm:px-6 lg:px-8">
+  <div class="sm:flex sm:items-center">
+    <div class="sm:flex-auto">
+      <h1 class="text-2xl font-semibold text-gray-900">Experiments</h1>
+      <p class="mt-2 text-sm text-gray-700">A list of all experiments in your account.</p>
+    </div>
+    <div class="mt-4 sm:mt-0 sm:ml-16 sm:flex-none">
+      <%= link_to 'New Experiment', new_experiment_path, class: 'btn btn-primary' %>
+    </div>
+  </div>
+  <div class="mt-8 flex flex-col">
+    <div class="-my-2 -mx-4 overflow-x-auto sm:-mx-6 lg:-mx-8">
+      <div class="inline-block min-w-full py-2 align-middle md:px-6 lg:px-8">
+        <div class="overflow-hidden shadow ring-1 ring-black ring-opacity-5 md:rounded-lg">
+          <table class="min-w-full divide-y divide-gray-300">
+            <thead class="bg-gray-50">
+              <tr>
+                <th scope="col" class="py-3.5 pl-4 pr-3 text-left text-sm font-semibold text-gray-900 sm:pl-6">Name</th>
+                <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">Status</th>
+                <th scope="col" class="px-3 py-3.5 text-left text-sm font-semibold text-gray-900">Dataset</th>
+                <th scope="col" class="relative py-3.5 pl-3 pr-4 sm:pr-6">
+                  <span class="sr-only">Actions</span>
+                </th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-gray-200 bg-white">
+              <% @experiments.each do |experiment| %>
+                <tr>
+                  <td class="whitespace-nowrap py-4 pl-4 pr-3 text-sm font-medium text-gray-900 sm:pl-6"><%= experiment.name %></td>
+                  <td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500"><%= experiment.status %></td>
+                  <td class="whitespace-nowrap px-3 py-4 text-sm text-gray-500"><%= experiment.dataset.name %></td>
+                  <td class="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-6">
+                    <%= link_to 'View', experiment_path(experiment), class: 'text-indigo-600 hover:text-indigo-900' %>
+                  </td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/leva/experiments/show.html.erb

@@ -0,0 +1,24 @@
+<% content_for :title, @experiment.name %>
+<div class="px-4 sm:px-6 lg:px-8">
+  <div class="sm:flex sm:items-center">
+    <div class="sm:flex-auto">
+      <h1 class="text-2xl font-semibold text-gray-900"><%= @experiment.name %></h1>
+      <p class="mt-2 text-sm text-gray-700">Status: <%= @experiment.status %></p>
+    </div>
+  </div>
+  <div class="mt-8 bg-white shadow overflow-hidden sm:rounded-lg">
+    <div class="px-4 py-5 sm:px-6">
+      <h3 class="text-lg leading-6 font-medium text-gray-900">Experiment Details</h3>
+    </div>
+    <div class="border-t border-gray-200 px-4 py-5 sm:p-0">
+      <dl class="sm:divide-y sm:divide-gray-200">
+        <div class="py-4 sm:py-5 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-6">
+          <dt class="text-sm font-medium text-gray-500">Dataset</dt>
+          <dd class="mt-1 text-sm text-gray-900 sm:mt-0 sm:col-span-2"><%= @experiment.dataset.name %></dd>
+        </div>
+        <!-- Add more experiment details here -->
+      </dl>
+    </div>
+  </div>
+  <!-- Add experiment results or other relevant information here -->
+</div>

data/app/views/leva/workbench/index.html.erb

@@ -0,0 +1,101 @@
+<% content_for :title, 'Workbench' %>
+<div class="flex h-[calc(100vh-4rem)] bg-gray-900 text-white">
+  <!-- Left Sidebar -->
+  <div class="w-64 h-full bg-gray-800 border-r border-gray-700 flex flex-col">
+    <div class="p-4">
+      <h2 class="text-xl font-bold mb-4">Prompts</h2>
+      <div class="space-y-2">
+        <% @prompts.each do |prompt| %>
+          <div class="bg-gray-700 p-2 rounded">
+            <span class="text-sm font-medium"><%= prompt.name %></span>
+            <span class="text-xs text-gray-400 ml-2">v<%= prompt.version %></span>
+          </div>
+        <% end %>
+      </div>
+    </div>
+    <div class="mt-auto p-4">
+      <%= link_to new_prompt_path, class: "w-full flex items-center justify-center space-x-2 bg-indigo-600 hover:bg-indigo-700 p-2 rounded" do %>
+        <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+          <path fill-rule="evenodd" d="M10 3a1 1 0 011 1v5h5a1 1 0 110 2h-5v5a1 1 0 11-2 0v-5H4a1 1 0 110-2h5V4a1 1 0 011-1z" clip-rule="evenodd" />
+        </svg>
+        <span>New Prompt</span>
+      <% end %>
+    </div>
+  </div>
+  <!-- Main Content -->
+  <div class="flex-1 flex flex-col">
+    <!-- Top Bar -->
+    <div class="bg-gray-800 p-4 flex items-center justify-between border-b border-gray-700">
+      <div>
+        <span class="font-medium"><%= @selected_prompt.name %></span>
+        <span class="text-xs text-gray-400 ml-2">v<%= @selected_prompt.version %></span>
+      </div>
+      <div class="flex items-center space-x-2">
+        <%= button_to run_workbench_index_path, method: :post, class: "flex items-center space-x-2 px-4 py-2 rounded bg-indigo-600 hover:bg-indigo-700" do %>
+          <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+            <path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM9.555 7.168A1 1 0 008 8v4a1 1 0 001.555.832l3-2a1 1 0 000-1.664l-3-2z" clip-rule="evenodd" />
+          </svg>
+          <span>Run</span>
+        <% end %>
+        <%= button_to run_with_evaluation_workbench_index_path, method: :post, class: "flex items-center space-x-2 px-4 py-2 rounded bg-green-600 hover:bg-green-700" do %>
+          <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+            <path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM9.555 7.168A1 1 0 008 8v4a1 1 0 001.555.832l3-2a1 1 0 000-1.664l-3-2z" clip-rule="evenodd" />
+          </svg>
+          <span>Run + Evaluate</span>
+        <% end %>
+      </div>
+    </div>
+    <!-- Scrollable Content -->
+    <div class="flex-1 flex overflow-hidden">
+      <div class="flex-1 overflow-y-auto p-6 space-y-4">
+        <!-- System Prompt -->
+        <div class="bg-gray-800 p-4 rounded">
+          <h2 class="text-sm font-semibold mb-2">SYSTEM PROMPT</h2>
+          <textarea
+            class="w-full bg-gray-700 text-white p-2 rounded resize-none"
+            style="height: auto; min-height: 100px;"
+            name="system_prompt"
+          ><%= @selected_prompt.system_prompt %></textarea>
+        </div>
+        <!-- User Message -->
+        <div class="bg-gray-800 p-4 rounded">
+          <h2 class="text-sm font-semibold mb-2">USER</h2>
+          <textarea
+            class="w-full bg-gray-700 text-white p-2 rounded resize-none"
+            style="height: auto; min-height: 200px;"
+            name="user_prompt"
+          ><%= @selected_prompt.user_prompt %></textarea>
+        </div>
+      </div>
+      <!-- Results Section -->
+      <div class="w-1/3 bg-gray-800 border-l border-gray-700 p-4 overflow-y-auto">
+        <h2 class="text-lg font-semibold mb-4">Results</h2>
+        <!-- Run Result -->
+        <div class="bg-gray-700 p-4 rounded mb-4">
+          <h3 class="text-sm font-semibold mb-2">Run Result</h3>
+          <p class="text-sm">
+            <%= flash[:notice] || "No results yet. Click 'Run' or 'Run + Evaluate' to start the analysis." %>
+          </p>
+        </div>
+        <!-- Evaluators -->
+        <div class="space-y-4">
+          <h3 class="text-sm font-semibold">Evaluators</h3>
+          <% @evaluators.each do |evaluator| %>
+            <div class="bg-gray-700 p-4 rounded">
+              <div class="flex items-center justify-between mb-2">
+                <span class="text-sm font-medium"><%= evaluator %></span>
+                <div class="flex items-center space-x-2">
+                  <%= button_to run_evaluator_workbench_index_path, method: :post, params: { evaluator: evaluator }, class: "p-1 bg-blue-500 hover:bg-blue-600 rounded" do %>
+                    <svg xmlns="http://www.w3.org/2000/svg" class="h-3 w-3" viewBox="0 0 20 20" fill="currentColor">
+                      <path fill-rule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM9.555 7.168A1 1 0 008 8v4a1 1 0 001.555.832l3-2a1 1 0 000-1.664l-3-2z" clip-rule="evenodd" />
+                    </svg>
+                  <% end %>
+                </div>
+              </div>
+            </div>
+          <% end %>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/leva/workbench/new.html.erb

@@ -0,0 +1,38 @@
+<% content_for :title, 'New Experiment' %>
+<div class="px-4 sm:px-6 lg:px-8">
+  <div class="sm:flex sm:items-center">
+    <div class="sm:flex-auto">
+      <h1 class="text-2xl font-semibold text-gray-900">New Experiment</h1>
+      <p class="mt-2 text-sm text-gray-700">Create a new experiment in your workbench.</p>
+    </div>
+  </div>
+  <div class="mt-8 max-w-xl">
+    <%= form_with(model: @experiment, url: experiments_path, local: true, class: "space-y-8 divide-y divide-gray-200") do |form| %>
+      <div class="space-y-8 divide-y divide-gray-200">
+        <div>
+          <div>
+            <h3 class="text-lg leading-6 font-medium text-gray-900">Experiment Information</h3>
+            <p class="mt-1 text-sm text-gray-500">Provide details for your new experiment.</p>
+          </div>
+          <div class="mt-6 grid grid-cols-1 gap-y-6 gap-x-4 sm:grid-cols-6">
+            <div class="sm:col-span-4">
+              <%= form.label :name, class: "block text-sm font-medium text-gray-700" %>
+              <div class="mt-1">
+                <%= form.text_field :name, class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>
+              </div>
+            </div>
+            <div class="sm:col-span-2">
+              <%= form.label :dataset_id, class: "block text-sm font-medium text-gray-700" %>
+              <div class="mt-1">
+                <%= form.select :dataset_id, options_for_select(@datasets.map { |dataset| [dataset.name, dataset.id] }), class: "shadow-sm focus:ring-indigo-500 focus:border-indigo-500 block w-full sm:text-sm border-gray-300 rounded-md" %>
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+      <div class="pt-5">
+        <%= form.submit class: "btn btn-primary btn-block" %>
+      </div>
+    <% end %>
+  </div>
+</div>

data/config/routes.rb

@@ -1,2 +1,12 @@
 Leva::Engine.routes.draw do
-end
+  root 'workbench#index'
+
+  resources :datasets
+  resources :experiments
+  resources :prompts
+  resources :workbench, only: [:index, :new, :show] do
+    post 'run', on: :collection
+    post 'run_with_evaluation', on: :collection
+    post 'run_evaluator', on: :collection
+  end
+end

data/db/migrate/20240813173222_create_leva_experiments.rb

@@ -3,7 +3,7 @@ class CreateLevaExperiments < ActiveRecord::Migration[7.2]
     create_table :leva_experiments do |t|
       t.string :name
       t.references :dataset, null: false, foreign_key: true
-      t.references :prompt, null: false, foreign_key: true
+      t.references :prompt, null: true, foreign_key: true
       t.integer :status
       t.text :metadata
 

data/lib/generators/leva/eval_generator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Leva
+  module Generators
+    class EvalGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates', __dir__)
+
+      def create_eval_file
+        template 'eval.rb.erb', File.join('app/evals', class_path, "#{file_name}_eval.rb")
+      end
+
+      private
+
+      def file_name
+        @_file_name ||= remove_possible_suffix(super)
+      end
+
+      def remove_possible_suffix(name)
+        name.sub(/_?eval$/i, '')
+      end
+    end
+  end
+end

data/lib/generators/leva/runner_generator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Leva
+  module Generators
+    class RunnerGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path('templates', __dir__)
+
+      def create_runner_file
+        template 'runner.rb.erb', File.join('app/runners', class_path, "#{file_name}_run.rb")
+      end
+
+      private
+
+      def file_name
+        @_file_name ||= remove_possible_suffix(super)
+      end
+
+      def remove_possible_suffix(name)
+        name.sub(/_?runner$/i, '')
+      end
+    end
+  end
+end

data/lib/generators/leva/templates/eval.rb.erb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class <%= class_name %>Eval < Leva::BaseEval
+  # @param prediction [String] The prediction to evaluate
+  # @param record [YourRecordClass] The record to evaluate
+  # @return [Leva::Result] The result of the evaluation
+  def evaluate(prediction, record)
+    # Implement your evaluation logic here
+
+    Leva::Result.new(
+      label: "<%= file_name.underscore %>",
+      score: score
+    )
+  end
+end

data/lib/generators/leva/templates/runner.rb.erb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class <%= class_name %>Run < Leva::BaseRun
+  # @param record [YourRecordClass] The record to run
+  # @return [String] The result of the run
+  def execute(record)
+    # Your model execution logic here
+    # This could involve calling an API, running a local model, etc.
+    # Return the result of the run to be used to evaluate the model
+  end
+end

data/lib/leva/base_eval.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Leva
+  class BaseEval
+    class << self
+      attr_reader :dataset_record_class_name
+
+      # Set the dataset record class for the eval
+      # @param class_name [String] The name of the dataset record class
+      def leva_dataset_record_class(class_name)
+        @dataset_record_class_name = class_name
+      end
+
+      # Run the experiment
+      # @param experiment [Leva::Experiment] The experiment to run
+      def run_experiment(experiment)
+        new.run_experiment(experiment)
+      end
+    end
+
+    # Run the experiment
+    # @param experiment [Leva::Experiment] The experiment to run
+    def run_experiment(experiment)
+      @experiment = experiment
+      @experiment.update(status: :running)
+
+      @experiment.dataset.records.each do |record|
+        @record = record
+        unless @record.class_name == self.class.dataset_record_class_name
+          raise ArgumentError, "Record class #{@record.class_name} does not match expected class #{self.class.dataset_record_class_name}"
+        end
+        ExperimentJob.perform_later(self, @record)
+      end
+
+      @experiment.update(status: :completed)
+    rescue StandardError => e
+      @experiment.update(status: :failed)
+      Rails.logger.error "Error in experiment #{@experiment.name}: #{e.message}"
+    end
+
+    # Run the evaluation for a single record
+    # @param record [ActiveRecord::Base] The record to evaluate
+    # @return [Leva::Result] The result of the evaluation
+    def run_each(record)
+      raise NotImplementedError, "Subclasses must implement the 'run_each' method"
+    end
+
+    # Save the result of an evaluation
+    # @param result [Leva::Result] The result of the evaluation
+    def save_result(result)
+      Leva::EvaluationResult.create!(
+        experiment: @experiment,
+        dataset_record: Leva::DatasetRecord.find_by(recordable: @record, dataset: @experiment.dataset),
+        prediction: result.prediction,
+        score: result.score,
+        label: result.label
+      )
+    end
+  end
+
+  # Represents the result of an evaluation
+  class Result
+    attr_reader :label, :prediction, :score
+
+    # Initialize a new Result
+    # @param label [String] The label for the result
+    # @param prediction [String] The prediction made by the evaluation
+    # @param score [Float] The score of the evaluation (0.0 to 1.0)
+    def initialize(label:, prediction:, score:)
+      @label = label
+      @prediction = prediction
+      @score = score
+    end
+  end
+end

data/lib/leva/version.rb

@@ -1,3 +1,3 @@
 module Leva
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/leva.rb

@@ -2,5 +2,93 @@ require "leva/version"
 require "leva/engine"
 
 module Leva
-  # Your code goes here...
-end
+  # Runs an evaluation experiment with the given run and evals.
+  #
+  # @param experiment [Leva::Experiment] The experiment to run.
+  # @param run [Leva::BaseRun] The run implementation to use.
+  # @param evals [Array<Leva::BaseEval>] The evaluation implementations to use.
+  # @return [void]
+  def self.run_evaluation(experiment:, run:, evals:)
+    results = run.run(experiment)
+    evals.each do |eval|
+      eval.evaluate_all(experiment, results)
+    end
+  end
+
+  # Base class for all run implementations in Leva.
+  #
+  # @abstract Subclass and override {#execute} to implement
+  #   custom run logic.
+  class BaseRun
+    # Executes the run on a given record.
+    #
+    # @param record [Leva::DatasetRecord] The record to run the model on.
+    # @return [Object] The output of the model execution.
+    # @raise [NotImplementedError] if the method is not implemented in a subclass.
+    def execute(record)
+      raise NotImplementedError, "#{self.class} must implement #execute"
+    end
+
+    # Runs the model on all records in an experiment.
+    #
+    # @param experiment [Leva::Experiment] The experiment to run.
+    # @return [Hash] A hash mapping dataset_record_ids to their execution results.
+    def run(experiment)
+      results = {}
+      experiment.dataset.dataset_records.find_each do |dataset_record|
+        result = execute(dataset_record.recordable)
+        results[dataset_record.id] = result
+      end
+      results
+    end
+  end
+
+  # Base class for all evaluation implementations in Leva.
+  #
+  # @abstract Subclass and override {#evaluate} to implement
+  #   custom evaluation logic.
+  class BaseEval
+    # Evaluates the model's prediction against the expected result.
+    #
+    # @param prediction [Object] The model's prediction.
+    # @param record [Object] The expected result.
+    # @return [Leva::Result] The evaluation result.
+    # @raise [NotImplementedError] if the method is not implemented in a subclass.
+    def evaluate(prediction, record)
+      raise NotImplementedError, "#{self.class} must implement #evaluate"
+    end
+
+    # Evaluates all results for an experiment.
+    #
+    # @param experiment [Leva::Experiment] The experiment to evaluate.
+    # @param results [Hash] A hash mapping dataset_record_ids to their execution results.
+    # @return [void]
+    def evaluate_all(experiment, results)
+      experiment.dataset.dataset_records.find_each do |dataset_record|
+        prediction = results[dataset_record.id]
+        evaluation = evaluate(prediction, dataset_record.recordable)
+
+        Leva::EvaluationResult.create!(
+          experiment: experiment,
+          dataset_record: dataset_record,
+          prediction: prediction,
+          score: evaluation.score,
+          label: evaluation.label
+        )
+      end
+    end
+  end
+
+  # Represents the result of an evaluation
+  class Result
+    attr_reader :label, :prediction, :score
+
+    # Initialize a new Result
+    # @param label [String] The label for the result
+    # @param score [Float] The score of the evaluation (0.0 to 1.0)
+    def initialize(label:, score:)
+      @label = label
+      @score = score
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: leva
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Kieran Klaassen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-13 00:00:00.000000000 Z
+date: 2024-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -39,8 +39,13 @@ files:
 - app/assets/config/leva_manifest.js
 - app/assets/stylesheets/leva/application.css
 - app/controllers/leva/application_controller.rb
+- app/controllers/leva/datasets_controller.rb
+- app/controllers/leva/experiments_controller.rb
+- app/controllers/leva/workbench_controller.rb
+- app/evals/test_sentiment_accuracy_eval.rb
 - app/helpers/leva/application_helper.rb
 - app/jobs/leva/application_job.rb
+- app/jobs/leva/experiment_job.rb
 - app/mailers/leva/application_mailer.rb
 - app/models/leva/application_record.rb
 - app/models/leva/dataset.rb
@@ -48,14 +53,26 @@ files:
 - app/models/leva/evaluation_result.rb
 - app/models/leva/experiment.rb
 - app/models/leva/prompt.rb
+- app/runners/test_sentiment_run.rb
 - app/views/layouts/leva/application.html.erb
+- app/views/leva/datasets/index.html.erb
+- app/views/leva/datasets/show.html.erb
+- app/views/leva/experiments/index.html.erb
+- app/views/leva/experiments/show.html.erb
+- app/views/leva/workbench/index.html.erb
+- app/views/leva/workbench/new.html.erb
 - config/routes.rb
 - db/migrate/20240813172916_create_leva_datasets.rb
 - db/migrate/20240813173033_create_leva_dataset_records.rb
 - db/migrate/20240813173050_create_leva_evaluation_results.rb
 - db/migrate/20240813173105_create_leva_prompts.rb
 - db/migrate/20240813173222_create_leva_experiments.rb
+- lib/generators/leva/eval_generator.rb
+- lib/generators/leva/runner_generator.rb
+- lib/generators/leva/templates/eval.rb.erb
+- lib/generators/leva/templates/runner.rb.erb
 - lib/leva.rb
+- lib/leva/base_eval.rb
 - lib/leva/engine.rb
 - lib/leva/version.rb
 - lib/tasks/auto_annotate_models.rake