leva 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d04a7b6d164d0962b4c9e2da411c8622f748688012e8724d95723787363531e
-  data.tar.gz: d2e199b18f717b368ed10b56f5a2bd5d52d26ffe738a670e570b167ced98461c
+  metadata.gz: 2abd3b8bab0e39683850e7e95df8815268332022c2adf6846cd27442e880cb88
+  data.tar.gz: 73c430d4a5a2c6a98dba70a36be988dd40541debbf10b274d62db8e232e931f7
 SHA512:
-  metadata.gz: 94fb2936e694272809dfe8e13fc052ead002a2cd29de4ff46ef4cb4bc1ecaf9b4fbab947fe43cb12c292f09badd23e8c3ca86bdb2e8e7d33c8221077611e35c2
-  data.tar.gz: 67e127f8b7147753021cb4ff96c2f3d5398a66490c702c2454df034d69cca1fec1153880713dfeec79e4f5898ffa44c3e187aa479b1ae2ea85926b1ea960b7be
+  metadata.gz: ee31e67dae95204cb6c6f2fc96ad41dae624ea48f6760b65571f8f1447fd24ca055a4dba33244067f0138db79a620ec82e8bbb3d3fd4363f9b6e98079b45b6a7
+  data.tar.gz: fadb2838c4d31e498f40ed8e5d20859c57e4e2df433fae1251d3215ddcbc3a2c4843b5a7bd0b73bb4ff4e2b3abb55842b83d7a018ad7feaa40a4da89e6313adc

data/README.md

@@ -27,14 +27,46 @@ rails db:migrate
 
 ### 1. Setting up Datasets
 
-First, create a dataset and add any ActiveRecord records you want to evaluate against:
+First, create a dataset and add any ActiveRecord records you want to evaluate against. To make your models compatible with Leva, include the `Leva::Recordable` concern in your model:
 
-```ruby
-dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset")
-dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive")
-dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative")
-dataset.add_record TextContent.create(text: "I's ok", expected_label: "Neutral")
-```
+````ruby
+class TextContent < ApplicationRecord
+  include Leva::Recordable
+
+  # @return [String] The ground truth label for the record
+  def ground_truth
+    expected_label
+  end
+
+  # @return [Hash] A hash of attributes to be displayed in the dataset records index
+  def index_attributes
+    {
+      text: text,
+      expected_label: expected_label,
+      created_at: created_at.strftime('%Y-%m-%d %H:%M:%S')
+    }
+  end
+
+  # @return [Hash] A hash of attributes to be displayed in the dataset record show view
+  def show_attributes
+    {
+      text: text,
+      expected_label: expected_label,
+      created_at: created_at.strftime('%Y-%m-%d %H:%M:%S')
+    }
+  end
+
+  # @return [Hash] A hash of attributes to be displayed in the dataset record show view
+  def to_llm_context
+    {
+      text: text,
+      expected_label: expected_label,
+      created_at: created_at.strftime('%Y-%m-%d %H:%M:%S')
+    }
+  end
+end
+
+dataset = Leva::Dataset.create(name: "Sentiment Analysis Dataset") dataset.add_record TextContent.create(text: "I love this product!", expected_label: "Positive") dataset.add_record TextContent.create(text: "Terrible experience", expected_label: "Negative") dataset.add_record TextContent.create(text: "It's ok", expected_label: "Neutral")
 
 ### 2. Implementing Runs
 
@@ -42,7 +74,7 @@ Create a run class to handle the execution of your inference logic:
 
 ```bash
 rails generate leva:runner sentiment
-```
+````
 
 ```ruby
 class SentimentRun < Leva::BaseRun
@@ -64,17 +96,17 @@ rails generate leva:eval sentiment_accuracy
 
 ```ruby
 class SentimentAccuracyEval < Leva::BaseEval
-  def evaluate(prediction, expected)
-    score = prediction == expected ? 1.0 : 0.0
-    Leva::Result.new(label: 'sentiment_accuracy', score: score)
+  def evaluate(prediction, record)
+    score = prediction == record.expected_label ? 1.0 : 0.0
+    [score, record.expected_label]
   end
 end
 
 class SentimentF1Eval < Leva::BaseEval
-  def evaluate(prediction, expected)
+  def evaluate(prediction, record)
     # Calculate F1 score
     # ...
-    Leva::Result.new(label: 'sentiment_f1', score: f1_score)
+    [f1_score, record.f1_score]
   end
 end
 ```
@@ -122,9 +154,9 @@ Leva.run_evaluation(experiment: experiment, run: run, evals: evals)
 After the experiments are complete, analyze the results:
 
 ```ruby
-experiment.evaluation_results.group_by(&:label).each do |label, results|
+experiment.evaluation_results.group_by(&:evaluator_class).each do |evaluator_class, results|
   average_score = results.average(&:score)
-  puts "#{label.capitalize} Average Score: #{average_score}"
+  puts "#{evaluator_class.capitalize} Average Score: #{average_score}"
 end
 ```
 
@@ -139,13 +171,13 @@ Ensure you set up any required API keys or other configurations in your Rails cr
 - `Leva`: Handles the process of running experiments.
 - `Leva::BaseRun`: Base class for run implementations.
 - `Leva::BaseEval`: Base class for eval implementations.
-- `Leva::Result`: Represents the result of an evaluation.
 
 ### Models
 
 - `Leva::Dataset`: Represents a collection of data to be evaluated.
 - `Leva::DatasetRecord`: Represents individual records within a dataset.
 - `Leva::Experiment`: Represents a single run of an evaluation on a dataset.
+- `Leva::RunnerResult`: Stores the results of each run execution.
 - `Leva::EvaluationResult`: Stores the results of each evaluation.
 - `Leva::Prompt`: Represents a prompt for an LLM.
 
@@ -156,3 +188,7 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/kieran
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Roadmap
+
+- [ ] Parallelize evaluation

data/app/controllers/leva/dataset_records_controller.rb

@@ -0,0 +1,21 @@
+module Leva
+  class DatasetRecordsController < ApplicationController
+    before_action :set_dataset
+
+    # GET /datasets/:dataset_id/records
+    def index
+      @records = @dataset.dataset_records.includes(:recordable)
+    end
+
+    # GET /datasets/:dataset_id/records/:id
+    def show
+      @record = @dataset.dataset_records.find(params[:id])
+    end
+
+    private
+
+    def set_dataset
+      @dataset = Dataset.find(params[:dataset_id])
+    end
+  end
+end

data/app/controllers/leva/datasets_controller.rb

@@ -13,6 +13,8 @@ module Leva
     # GET /datasets/1
     # @return [void]
     def show
+      @experiments = @dataset.experiments
+      @new_experiment = Experiment.new(dataset: @dataset)
     end
 
     # GET /datasets/new
@@ -24,6 +26,7 @@ module Leva
     # GET /datasets/1/edit
     # @return [void]
     def edit
+      # The @dataset is already set by the before_action
     end
 
     # POST /datasets
@@ -51,8 +54,12 @@ module Leva
     # DELETE /datasets/1
     # @return [void]
     def destroy
-      @dataset.destroy
-      redirect_to datasets_url, notice: 'Dataset was successfully destroyed.'
+      if @dataset.dataset_records.any?
+        redirect_to @dataset, alert: 'Cannot delete dataset with existing records.'
+      else
+        @dataset.destroy
+        redirect_to datasets_url, notice: 'Dataset was successfully destroyed.'
+      end
     end
 
     private

data/app/controllers/leva/experiments_controller.rb

@@ -2,7 +2,11 @@
 
 module Leva
   class ExperimentsController < ApplicationController
-    before_action :set_experiment, only: [:show, :edit, :update, :destroy]
+    include ApplicationHelper
+
+    before_action :set_experiment, only: [:show, :edit, :update]
+    before_action :check_editable, only: [:edit, :update]
+    before_action :load_runners_and_evaluators, only: [:new, :edit, :create, :update]
 
     # GET /experiments
     # @return [void]
@@ -13,17 +17,19 @@ module Leva
     # GET /experiments/1
     # @return [void]
     def show
+      @experiment = Experiment.includes(runner_results: :evaluation_results).find(params[:id])
     end
 
     # GET /experiments/new
     # @return [void]
     def new
-      @experiment = Experiment.new
+      @experiment = Experiment.new(dataset_id: params[:dataset_id])
     end
 
     # GET /experiments/1/edit
     # @return [void]
     def edit
+      # The @experiment is already set by the before_action
     end
 
     # POST /experiments
@@ -32,8 +38,8 @@ module Leva
       @experiment = Experiment.new(experiment_params)
 
       if @experiment.save
-        ExperimentJob.perform_later(@experiment)
-        redirect_to @experiment, notice: 'Experiment was successfully created.'
+        ExperimentJob.perform_later(@experiment) unless @experiment.completed?
+        redirect_to @experiment, notice: 'Experiment was successfully created and is now running.'
       else
         render :new
       end
@@ -49,11 +55,21 @@ module Leva
       end
     end
 
-    # DELETE /experiments/1
+    # POST /experiments/1/rerun
     # @return [void]
-    def destroy
-      @experiment.destroy
-      redirect_to experiments_url, notice: 'Experiment was successfully destroyed.'
+    def rerun
+      @experiment = Experiment.find(params[:id])
+
+      # Delete existing runner results and evaluation results
+      @experiment.runner_results.destroy_all
+
+      # Reset experiment status to pending
+      @experiment.update(status: :pending)
+
+      # Queue the job again
+      ExperimentJob.perform_later(@experiment)
+
+      redirect_to @experiment, notice: 'Experiment has been reset and is now running again.'
     end
 
     private
@@ -67,7 +83,16 @@ module Leva
     # Only allow a list of trusted parameters through.
     # @return [ActionController::Parameters]
     def experiment_params
-      params.require(:experiment).permit(:name, :description, :dataset_id)
+      params.require(:experiment).permit(:name, :description, :dataset_id, :prompt_id, :runner_class, evaluator_classes: [])
+    end
+
+    def load_runners_and_evaluators
+      @runners = load_runners
+      @evaluators = load_evaluators
+    end
+
+    def check_editable
+      redirect_to @experiment, alert: 'Completed experiments cannot be edited.' if @experiment.completed?
     end
   end
 end

data/app/controllers/leva/runner_results_controller.rb

@@ -0,0 +1,8 @@
+module Leva
+  class RunnerResultsController < ApplicationController
+    def show
+      @experiment = Experiment.find(params[:experiment_id])
+      @runner_result = @experiment.runner_results.find(params[:id])
+    end
+  end
+end

data/app/controllers/leva/workbench_controller.rb

@@ -2,39 +2,112 @@
 
 module Leva
   class WorkbenchController < ApplicationController
+    include ApplicationHelper
+
+    before_action :set_prompt, only: [:index, :edit, :update, :run, :run_all_evals, :run_evaluator]
+    before_action :set_dataset_record, only: [:index, :run, :run_all_evals, :run_evaluator]
+    before_action :set_runner_result, only: [:index, :run_all_evals, :run_evaluator]
+
     # GET /workbench
     # @return [void]
     def index
       @prompts = Prompt.all
-      @selected_prompt = Prompt.first || Prompt.create!(name: "Test Prompt", version: 1, system_prompt: "You are a helpful assistant.", user_prompt: "Hello, how can I help you today?")
-      @evaluators = ['Evaluator 1', 'Evaluator 2', 'Evaluator 3']
+      @selected_prompt = @prompt || Prompt.first
+      @evaluators = load_evaluators
+      @runners = load_runners
+      @selected_runner = params[:runner] || @runners.first&.name
+      @selected_dataset_record = params[:dataset_record_id] || DatasetRecord.first&.id
     end
 
     # GET /workbench/new
     # @return [void]
     def new
-      @experiment = Experiment.new
+      @prompt = Prompt.new
+      @predefined_prompts = load_predefined_prompts
+    end
+
+    # POST /workbench
+    # @return [void]
+    def create
+      @prompt = Prompt.new(prompt_params)
+      if @prompt.save
+        redirect_to workbench_index_path(prompt_id: @prompt.id), notice: 'Prompt was successfully created.'
+      else
+        render :new
+      end
     end
 
     # GET /workbench/1
     # @return [void]
-    def show
-      @experiment = Experiment.find(params[:id])
+    def edit
+    end
+
+    # PATCH/PUT /workbench/1
+    # @return [void]
+    def update
+      @prompt = Prompt.find(params[:id])
+      if @prompt.update(prompt_params)
+        render json: { status: 'success', message: 'Prompt updated successfully' }
+      else
+        render json: { status: 'error', errors: @prompt.errors.full_messages }, status: :unprocessable_entity
+      end
     end
 
     def run
-      # Implement the logic for running the prompt
-      redirect_to workbench_index_path, notice: 'Prompt run successfully'
+      return redirect_to workbench_index_path, alert: 'Please select a record and a runner' unless @dataset_record && run_params[:runner]
+
+      runner_class = run_params[:runner].constantize
+      return redirect_to workbench_index_path, alert: 'Invalid runner selected' unless runner_class < Leva::BaseRun
+
+      runner = runner_class.new
+      runner_result = runner.execute_and_store(nil, @dataset_record, @prompt)
+
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: run_params[:runner]), notice: 'Run completed successfully'
     end
 
-    def run_with_evaluation
-      # Implement the logic for running the prompt with evaluation
-      redirect_to workbench_index_path, notice: 'Prompt run with evaluation successfully'
+    def run_all_evals
+      return redirect_to workbench_index_path, alert: 'No runner result available' unless @runner_result
+
+      load_evaluators.each do |evaluator_class|
+        evaluator = evaluator_class.new
+        evaluator.evaluate_and_store(nil, @runner_result)
+      end
+
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: 'All evaluations completed successfully'
     end
 
     def run_evaluator
-      # Implement the logic for running a single evaluator
-      redirect_to workbench_index_path, notice: 'Evaluator run successfully'
+      return redirect_to workbench_index_path, alert: 'No runner result available' unless @runner_result
+
+      evaluator_class = params[:evaluator].constantize
+      return redirect_to workbench_index_path, alert: 'Invalid evaluator selected' unless evaluator_class < Leva::BaseEval
+
+      evaluator = evaluator_class.new
+      evaluator.evaluate_and_store(nil, @runner_result)
+
+      redirect_to workbench_index_path(prompt_id: @prompt.id, dataset_record_id: @dataset_record.id, runner: params[:runner]), notice: 'Evaluator run successfully'
+    end
+
+    private
+
+    def set_prompt
+      @prompt = params[:prompt_id] ? Prompt.find(params[:prompt_id]) : Prompt.first
+    end
+
+    def prompt_params
+      params.require(:prompt).permit(:name, :system_prompt, :user_prompt, :version)
+    end
+
+    def set_dataset_record
+      @dataset_record = DatasetRecord.find_by(id: params[:dataset_record_id]) || DatasetRecord.first
+    end
+
+    def run_params
+      params.permit(:runner, :prompt_id, :dataset_record_id)
+    end
+
+    def set_runner_result
+      @runner_result = @dataset_record.runner_results.last if @dataset_record
     end
   end
 end

data/app/helpers/leva/application_helper.rb

@@ -1,4 +1,43 @@
 module Leva
   module ApplicationHelper
+    # Loads all evaluator classes that inherit from Leva::BaseEval
+    #
+    # @return [Array<Class>] An array of evaluator classes
+    def load_evaluators
+      load_classes_from_directory('app/evals', Leva::BaseEval) || []
+    end
+
+    # Loads all runner classes that inherit from Leva::BaseRun
+    #
+    # @return [Array<Class>] An array of runner classes
+    def load_runners
+      load_classes_from_directory('app/runners', Leva::BaseRun) || []
+    end
+
+    # Loads predefined prompts from markdown files
+    #
+    # @return [Array<Array<String, String>>] An array of prompt name and content pairs
+    def load_predefined_prompts
+      prompts = Dir.glob(Rails.root.join('app', 'prompts', '*.md')).map do |file|
+        name = File.basename(file, '.md').titleize
+        content = File.read(file)
+        [name, content]
+      end
+      prompts
+    end
+
+    private
+
+    # Loads classes from a specified directory that inherit from a given base class
+    #
+    # @param directory [String] The directory path to load classes from
+    # @param base_class [Class] The base class that loaded classes should inherit from
+    # @return [Array<Class>] An array of loaded classes
+    def load_classes_from_directory(directory, base_class)
+      classes = Dir[Rails.root.join(directory, '*.rb')].map do |file|
+        File.basename(file, '.rb').camelize.constantize
+      end.select { |klass| klass < base_class }
+      classes.empty? ? [] : classes
+    end
   end
 end

data/app/javascript/controllers/prompt_form_controller.js

@@ -0,0 +1,45 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["form"];
+
+  autoSave() {
+    clearTimeout(this.timeout);
+    this.timeout = setTimeout(() => {
+      this.submitForm();
+    }, 500);
+  }
+
+  submitForm() {
+    const form = this.element;
+    const formData = new FormData(form);
+
+    fetch(form.action, {
+      method: form.method,
+      body: formData,
+      headers: {
+        Accept: "application/json",
+        "X-CSRF-Token": document.querySelector('meta[name="csrf-token"]').content,
+      },
+    })
+      .then((response) => response.json())
+      .then((data) => {
+        const statusElement = document.getElementById("form-status");
+        if (data.status === "success") {
+          statusElement.textContent = "Changes saved successfully";
+          statusElement.classList.add("text-green-500");
+          statusElement.classList.remove("text-red-500");
+        } else {
+          statusElement.textContent = `Error: ${data.errors.join(", ")}`;
+          statusElement.classList.add("text-red-500");
+          statusElement.classList.remove("text-green-500");
+        }
+        setTimeout(() => {
+          statusElement.textContent = "";
+        }, 3000);
+      })
+      .catch((error) => {
+        console.error("Error:", error);
+      });
+  }
+}

data/app/javascript/controllers/prompt_selector_controller.js

@@ -0,0 +1,31 @@
+import { Controller } from "@hotwired/stimulus";
+
+export default class extends Controller {
+  static targets = ["userPromptField"];
+
+  toggleUserPrompt(event) {
+    const selectedFile = event.target.value;
+    if (selectedFile) {
+      this.userPromptFieldTarget.style.display = "none";
+      this.loadPredefinedPrompt(selectedFile);
+    } else {
+      this.userPromptFieldTarget.style.display = "block";
+      this.clearUserPrompt();
+    }
+  }
+
+  loadPredefinedPrompt(file) {
+    fetch(file)
+      .then((response) => response.text())
+      .then((content) => {
+        const userPromptTextarea = this.userPromptFieldTarget.querySelector("textarea");
+        userPromptTextarea.value = content;
+      })
+      .catch((error) => console.error("Error loading predefined prompt:", error));
+  }
+
+  clearUserPrompt() {
+    const userPromptTextarea = this.userPromptFieldTarget.querySelector("textarea");
+    userPromptTextarea.value = "";
+  }
+}

data/app/jobs/leva/experiment_job.rb

@@ -4,13 +4,18 @@ module Leva
   class ExperimentJob < ApplicationJob
     queue_as :default
 
-    # Perform the experiment
+    # Perform the experiment by scheduling all dataset records for evaluation
     #
     # @param experiment [Experiment] The experiment to run
     # @return [void]
-    def perform(eval, record)
-      result = eval.run_each(record)
-      eval.save_result(result)
+    def perform(experiment)
+      return if experiment.completed? || experiment.running?
+
+      experiment.update!(status: :running)
+
+      experiment.dataset.dataset_records.each_with_index do |record, index|
+        RunEvalJob.set(wait: 3.seconds * index).perform_later(experiment.id, record.id)
+      end
     end
   end
 end

data/app/jobs/leva/run_eval_job.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Leva
+  class RunEvalJob < ApplicationJob
+    queue_as :default
+
+    # Perform a single run and evaluation for a dataset record
+    #
+    # @param experiment_id [Integer] The ID of the experiment
+    # @param dataset_record_id [Integer] The ID of the dataset record
+    # @return [void]
+    def perform(experiment_id, dataset_record_id)
+      experiment = Experiment.find(experiment_id)
+      dataset_record = DatasetRecord.find(dataset_record_id)
+
+      run = constantize_class(experiment.runner_class).new
+      evals = experiment.evaluator_classes.compact.reject(&:empty?).map { |klass| constantize_class(klass).new }
+
+      Leva.run_single_evaluation(experiment: experiment, run: run, evals: evals, dataset_record: dataset_record)
+
+      experiment.update!(status: :completed) if is_last(experiment)
+    end
+
+    private
+
+    def constantize_class(class_name)
+      class_name.constantize
+    rescue NameError => e
+      raise NameError, "Invalid class name: #{class_name}. Error: #{e.message}"
+    end
+
+    # Check if all dataset records for the experiment have a runner result
+    #
+    # @param experiment [Experiment] The experiment to check
+    # @return [Boolean] True if all dataset records have a runner result, false otherwise
+    def is_last(experiment)
+      experiment.dataset.dataset_records.count == experiment.runner_results.count
+    end
+  end
+end

data/app/models/concerns/leva/recordable.rb

@@ -0,0 +1,37 @@
+module Leva
+  module Recordable
+    extend ActiveSupport::Concern
+
+    included do
+      has_many :dataset_records, as: :recordable, class_name: 'Leva::DatasetRecord', dependent: :destroy
+      has_many :datasets, through: :dataset_records, class_name: 'Leva::Dataset'
+      has_many :runner_results, through: :dataset_records, class_name: 'Leva::RunnerResult'
+      has_many :evaluation_results, through: :runner_results, class_name: 'Leva::EvaluationResult'
+    end
+
+    # @return [String] The ground truth label for the record
+    def ground_truth
+      raise NotImplementedError, "#{self.class} must implement #ground_truth"
+    end
+
+    # @return [Hash] A hash of attributes to be displayed in the dataset records index
+    def index_attributes
+      raise NotImplementedError, "#{self.class} must implement #index_attributes"
+    end
+
+    # @return [Hash] A hash of attributes to be displayed in the dataset record show view
+    def show_attributes
+      raise NotImplementedError, "#{self.class} must implement #show_attributes"
+    end
+
+    # @return [Hash] A hash of attributes to be liquified for LLM context
+    def to_llm_context
+      raise NotImplementedError, "#{self.class} must implement #to_llm_context"
+    end
+
+    # @return [Regexp] A regex pattern to extract the contents of a LLM response
+    def extract_regex_pattern
+      false
+    end
+  end
+end

data/app/models/leva/dataset.rb

@@ -2,18 +2,27 @@
 #
 # Table name: leva_datasets
 #
-#  id         :integer          not null, primary key
-#  name       :string
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
+#  id          :integer          not null, primary key
+#  description :text
+#  name        :string
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
 #
 module Leva
   class Dataset < ApplicationRecord
     has_many :dataset_records, dependent: :destroy
     has_many :experiments, dependent: :destroy
 
+    validates :name, presence: true
+
+    # Adds a record to the dataset if it doesn't already exist
+    #
+    # @param record [ActiveRecord::Base] The record to be added to the dataset
+    # @return [Leva::DatasetRecord, nil] The created dataset record or nil if it already exists
     def add_record(record)
-      dataset_records.create(recordable: record)
+      dataset_records.find_or_create_by(recordable: record) do |dr|
+        dr.recordable = record
+      end
     end
   end
-end
+end