leva 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91394838ea4f4d47fd0968806beee6ce6d30ecc11888565630d76ca0a6d48fc0
-  data.tar.gz: 48f3178cf8be1f4fafaf9e485b5e6bab93c54570f71d738623e19d15cb6b2a13
+  metadata.gz: '03694d16308b610d8c1cc83ec070cf2c0a03273d93b4e220834ff063f8df5b0a'
+  data.tar.gz: 31fa8e5737410dbb9b5729bf43616ef037fbad1c6b8188e60649a5156c8f87c1
 SHA512:
-  metadata.gz: e3f8f96aef6161234ceff59cfb81f18d650b4e617aafcbd53badca722720fd065b206a10b3319bd6569c07d1d71ae50398b35081a526c242f691239421200262
-  data.tar.gz: 6f33990542e0a0d4c64611fdd5bab3ff6af7b42838350a414801a997ce5aefc72ce54f7c40ee6b1977fd6523616fc18c1d4a036ff02f44e82424b74acf4ad8e9
+  metadata.gz: f12f9ec8d00a5dcd9a8c003a598d9ec316be4bd8b8b2deb7a99680a14dcd64790b496829e7635e28f5b86dd7a5f484b9043b504bda24f7e3d0fd75b8e4eee271
+  data.tar.gz: 293f53edc39d95ed612b0ce0e0e5097f38e888990c7e8530b54da6afcf2015ae7f150f8f9bd9d2bb1171c5bf18c0c4a34180482594c376ed17341ae42bce9f09

data/README.md

@@ -181,6 +181,60 @@ experiment.evaluation_results.group_by(&:evaluator_class).each do |evaluator_cla
 end
 ```
 
+## Prompt Optimization (DSPy Integration)
+
+Leva includes optional prompt optimization powered by [DSPy.rb](https://github.com/kieranklaassen/dspy.rb). This feature automatically finds optimal prompts and few-shot examples for your datasets.
+
+**Requirements:**
+- Ruby 3.3.0 or higher
+- DSPy gem and optional optimizer gems
+
+### Installation
+
+Add the DSPy gems to your Gemfile:
+
+```ruby
+gem "dspy"           # Core DSPy functionality (required)
+gem "dspy-gepa"      # GEPA optimizer (optional, recommended)
+gem "dspy-miprov2"   # MIPROv2 optimizer (optional)
+```
+
+### Available Optimizers
+
+| Optimizer | Best For | Description |
+|-----------|----------|-------------|
+| **Bootstrap** | Quick iteration, small datasets | Fast selection of few-shot examples. No gem required. |
+| **GEPA** | Maximum quality | State-of-the-art reflective prompt evolution. 10-14% better than MIPROv2. |
+| **MIPROv2** | Large datasets (200+) | Bayesian optimization for instructions and examples. |
+
+### Usage
+
+```ruby
+# Create an optimizer for your dataset
+optimizer = Leva::PromptOptimizer.new(
+  dataset: dataset,
+  optimizer: :gepa,      # :bootstrap, :gepa, or :miprov2
+  mode: :medium,         # :light, :medium, or :heavy
+  model: "gpt-4o-mini"   # Any model supported by RubyLLM
+)
+
+# Run optimization
+result = optimizer.optimize
+
+# Result contains optimized prompts
+result[:system_prompt]  # Optimized instruction
+result[:user_prompt]    # Template with Liquid variables
+result[:metadata]       # Score, examples, and optimization details
+```
+
+### Optimization Modes
+
+| Mode | Duration | Use Case |
+|------|----------|----------|
+| `:light` | ~5 min | Quick experiments |
+| `:medium` | ~15 min | Balanced quality/speed |
+| `:heavy` | ~30 min | Production prompts |
+
 ## Configuration
 
 Ensure you set up any required API keys or other configurations in your Rails credentials or environment variables.

data/app/assets/stylesheets/leva/application.css

@@ -947,6 +947,15 @@ strong {
   color: var(--info-400);
 }
 
+.badge-optimized {
+  background: rgba(168, 127, 219, 0.15);
+  color: #b794f4;
+  font-size: 0.65rem;
+  padding: 2px 6px;
+  margin-left: 6px;
+  vertical-align: middle;
+}
+
 /* Status Dots */
 .status-dot {
   display: inline-block;

data/app/controllers/leva/dataset_optimizations_controller.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Leva
+  class DatasetOptimizationsController < ApplicationController
+    before_action :set_dataset
+
+    # GET /datasets/:dataset_id/optimization/new
+    # Shows the prompt optimization form
+    # @return [void]
+    def new
+      @record_count = @dataset.dataset_records.count
+      @prompt_optimizer = PromptOptimizer.new(dataset: @dataset)
+      @can_optimize = @prompt_optimizer.can_optimize?
+      @records_needed = @prompt_optimizer.records_needed
+      @modes = PromptOptimizer::MODES
+      @models = PromptOptimizer.available_models
+      @optimizers = PromptOptimizer::OPTIMIZERS
+    end
+
+    # POST /datasets/:dataset_id/optimization
+    # Starts the prompt optimization job with progress tracking
+    # @return [void]
+    def create
+      opt_params = optimization_params
+
+      @optimization_run = @dataset.optimization_runs.create!(
+        prompt_name: opt_params[:prompt_name],
+        mode: opt_params[:mode],
+        model: opt_params[:model],
+        optimizer: opt_params[:optimizer],
+        status: :pending
+      )
+
+      PromptOptimizationJob.perform_later(optimization_run_id: @optimization_run.id)
+
+      redirect_to optimization_run_path(@optimization_run)
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    # @return [void]
+    def set_dataset
+      @dataset = Dataset.find(params[:dataset_id])
+    end
+
+    # Strong parameters for optimization run creation.
+    # @return [Hash]
+    # @raise [ActionController::BadRequest] If model is invalid
+    def optimization_params
+      model = params[:model].presence || PromptOptimizer::DEFAULT_MODEL
+      unless PromptOptimizer.find_model(model)
+        raise ActionController::BadRequest, "Invalid model: #{model}"
+      end
+
+      {
+        prompt_name: params[:prompt_name].presence || "Optimized: #{@dataset.name}",
+        mode: params[:mode].presence || "light",
+        model: model,
+        optimizer: params[:optimizer].presence || PromptOptimizer::DEFAULT_OPTIMIZER.to_s
+      }
+    end
+  end
+end

data/app/controllers/leva/experiments_controller.rb

@@ -2,8 +2,6 @@
 
 module Leva
   class ExperimentsController < ApplicationController
-    include ApplicationHelper
-
     before_action :set_experiment, only: [ :show, :edit, :update ]
     before_action :check_editable, only: [ :edit, :update ]
     before_action :load_runners_and_evaluators, only: [ :new, :edit, :create, :update ]
@@ -11,7 +9,8 @@ module Leva
     # GET /experiments
     # @return [void]
     def index
-      @experiments = Experiment.all
+      @experiments = Experiment.includes(:evaluation_results).all
+      @evaluator_classes = Leva::EvaluationResult.distinct.pluck(:evaluator_class)
     end
 
     # GET /experiments/1
@@ -83,12 +82,21 @@ module Leva
     # Only allow a list of trusted parameters through.
     # @return [ActionController::Parameters]
     def experiment_params
-      params.require(:experiment).permit(:name, :description, :dataset_id, :prompt_id, :runner_class, evaluator_classes: [])
+      permitted = params.require(:experiment).permit(:name, :description, :dataset_id, :prompt_id, :runner_class, evaluator_classes: [], metadata: {})
+      # Ensure metadata is a hash, not ActionController::Parameters
+      if permitted[:metadata].present?
+        metadata_hash = permitted[:metadata].to_h
+        if metadata_hash.to_json.bytesize > 100_000
+          raise ActionController::BadRequest, "Metadata exceeds maximum size of 100KB"
+        end
+        permitted[:metadata] = metadata_hash
+      end
+      permitted
     end
 
     def load_runners_and_evaluators
-      @runners = load_runners
-      @evaluators = load_evaluators
+      @runners = Leva::ClassLoader.runners
+      @evaluators = Leva::ClassLoader.evaluators
     end
 
     def check_editable

data/app/controllers/leva/workbench_controller.rb

@@ -19,14 +19,12 @@ module Leva
       @selected_dataset_record = params[:dataset_record_id] || DatasetRecord.first&.id
 
       # Get merged context if runner and dataset record are available
-      if @selected_runner && @dataset_record
-        runner_class = @selected_runner.constantize rescue nil
-        if runner_class && runner_class < Leva::BaseRun
-          runner = runner_class.new
-          @record_context = @dataset_record.recordable.to_llm_context
-          @runner_context = runner.to_llm_context(@dataset_record.recordable)
-          @merged_context = @record_context.merge(@runner_context)
-        end
+      if @selected_runner && @dataset_record && valid_runner?(@selected_runner)
+        runner_class = @selected_runner.constantize
+        runner = runner_class.new
+        @record_context = @dataset_record.recordable.to_llm_context
+        @runner_context = runner.to_llm_context(@dataset_record.recordable)
+        @merged_context = @record_context.merge(@runner_context)
       end
     end
 
@@ -67,8 +65,8 @@ module Leva
     def run
       return redirect_to workbench_index_path, alert: "Please select a record and a runner" unless @dataset_record && run_params[:runner]
 
+      return redirect_to workbench_index_path, alert: "Invalid runner selected" unless valid_runner?(run_params[:runner])
       runner_class = run_params[:runner].constantize
-      return redirect_to workbench_index_path, alert: "Invalid runner selected" unless runner_class < Leva::BaseRun
 
       runner = runner_class.new
       runner_result = runner.execute_and_store(nil, @dataset_record, @prompt)
@@ -90,8 +88,8 @@ module Leva
     def run_evaluator
       return redirect_to workbench_index_path, alert: "No runner result available" unless @runner_result
 
+      return redirect_to workbench_index_path, alert: "Invalid evaluator selected" unless allowed_evaluator_names.include?(params[:evaluator])
       evaluator_class = params[:evaluator].constantize
-      return redirect_to workbench_index_path, alert: "Invalid evaluator selected" unless evaluator_class < Leva::BaseEval
 
       evaluator = evaluator_class.new
       evaluator.evaluate_and_store(nil, @runner_result)
@@ -120,5 +118,23 @@ module Leva
     def set_runner_result
       @runner_result = @dataset_record.runner_results.last if @dataset_record
     end
+
+    def allowed_runner_names
+      @allowed_runner_names ||= load_runners.map(&:name)
+    end
+
+    def allowed_evaluator_names
+      @allowed_evaluator_names ||= load_evaluators.map(&:name)
+    end
+
+    def valid_runner?(runner_name)
+      return true if allowed_runner_names.include?(runner_name)
+
+      # Also accept any class that inherits from BaseRun (for testing)
+      klass = runner_name.constantize
+      klass < Leva::BaseRun
+    rescue NameError
+      false
+    end
   end
 end

data/app/helpers/leva/application_helper.rb

@@ -4,14 +4,44 @@ module Leva
     #
     # @return [Array<Class>] An array of evaluator classes
     def load_evaluators
-      load_classes_from_directory("app/evals", Leva::BaseEval) || []
+      Leva::ClassLoader.evaluators
     end
 
     # Loads all runner classes that inherit from Leva::BaseRun
     #
     # @return [Array<Class>] An array of runner classes
     def load_runners
-      load_classes_from_directory("app/runners", Leva::BaseRun) || []
+      Leva::ClassLoader.runners
+    end
+
+    # Returns the CSS class for a score value.
+    #
+    # @param score [Float, nil] The score value (0.0 - 1.0)
+    # @return [String] The CSS class for the score
+    def score_class(score)
+      return "" if score.nil?
+
+      case score
+      when 0...0.2 then "score-bad"
+      when 0.2...0.4 then "score-poor"
+      when 0.4...0.6 then "score-fair"
+      when 0.6...0.8 then "score-good"
+      else "score-excellent"
+      end
+    end
+
+    # Returns the display name for a model.
+    #
+    # Uses RubyLLM to find the model and get its display name,
+    # falling back to extracting the name from the model ID.
+    #
+    # @param model_id [String] The model ID
+    # @return [String] The display name for the model
+    def model_display_name(model_id)
+      return "—" if model_id.blank?
+
+      @models_cache ||= Leva::PromptOptimizer.available_models.index_by(&:id)
+      @models_cache[model_id]&.name || model_id.split("/").last
     end
 
     # Loads predefined prompts from markdown files
@@ -25,19 +55,5 @@ module Leva
       end
       prompts
     end
-
-    private
-
-    # Loads classes from a specified directory that inherit from a given base class
-    #
-    # @param directory [String] The directory path to load classes from
-    # @param base_class [Class] The base class that loaded classes should inherit from
-    # @return [Array<Class>] An array of loaded classes
-    def load_classes_from_directory(directory, base_class)
-      classes = Dir[Rails.root.join(directory, "*.rb")].map do |file|
-        File.basename(file, ".rb").camelize.constantize
-      end.select { |klass| klass < base_class }
-      classes.empty? ? [] : classes
-    end
   end
 end

data/app/models/leva/dataset.rb

@@ -12,6 +12,7 @@ module Leva
   class Dataset < ApplicationRecord
     has_many :dataset_records, dependent: :destroy
     has_many :experiments, dependent: :destroy
+    has_many :optimization_runs, dependent: :destroy
 
     validates :name, presence: true
 

data/app/models/leva/experiment.rb

@@ -39,5 +39,6 @@ module Leva
     enum :status, { pending: 0, running: 1, completed: 2, failed: 3 }, default: :pending
 
     serialize :evaluator_classes, coder: JSON, type: Array
+    serialize :metadata, coder: JSON
   end
 end

data/app/models/leva/optimization_run.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Leva
+  # Tracks the progress and status of prompt optimization runs.
+  #
+  # @example Create and track an optimization run
+  #   run = OptimizationRun.create!(
+  #     dataset: dataset,
+  #     prompt_name: "My Optimized Prompt",
+  #     mode: "light"
+  #   )
+  #   run.start!
+  #   run.update_progress(step: "bootstrapping", progress: 50, examples_processed: 5)
+  #   run.complete!(prompt)
+  class OptimizationRun < ApplicationRecord
+    self.table_name = "leva_optimization_runs"
+
+    belongs_to :dataset
+    belongs_to :prompt, optional: true
+
+    enum :status, {
+      pending: "pending",
+      running: "running",
+      completed: "completed",
+      failed: "failed"
+    }, default: :pending
+
+    validates :prompt_name, presence: true, length: { maximum: 255 }
+    validates :mode, presence: true, inclusion: { in: %w[light medium heavy] }
+    validates :model, presence: true
+    validates :optimizer, inclusion: { in: PromptOptimizer::OPTIMIZERS.keys.map(&:to_s) }
+    validates :progress, numericality: { in: 0..100 }
+
+    # Defined optimization steps for display
+    STEPS = {
+      "validating" => { label: "Validating dataset", icon: "check" },
+      "splitting_data" => { label: "Splitting data", icon: "scissors" },
+      "generating_signature" => { label: "Generating signature", icon: "code" },
+      "bootstrapping" => { label: "Bootstrapping examples", icon: "zap" },
+      "evaluating" => { label: "Evaluating results", icon: "bar-chart" },
+      "building_result" => { label: "Building prompt", icon: "package" },
+      "complete" => { label: "Complete", icon: "check-circle" }
+    }.freeze
+
+    # Marks the run as started.
+    #
+    # @return [void]
+    def start!
+      update!(status: :running, current_step: "validating", progress: 0)
+    end
+
+    # Updates the progress of the optimization run.
+    #
+    # @param step [String] Current step name
+    # @param progress [Integer] Progress percentage (0-100)
+    # @param examples_processed [Integer, nil] Number of examples processed
+    # @param total [Integer, nil] Total examples to process
+    # @return [void]
+    def update_progress(step:, progress:, examples_processed: nil, total: nil)
+      attrs = { current_step: step, progress: progress }
+      attrs[:examples_processed] = examples_processed if examples_processed
+      attrs[:total_examples] = total if total
+      update!(attrs)
+    end
+
+    # Marks the run as completed with the created prompt.
+    #
+    # @param created_prompt [Leva::Prompt] The optimized prompt
+    # @return [void]
+    def complete!(created_prompt)
+      update!(
+        status: :completed,
+        prompt: created_prompt,
+        current_step: "complete",
+        progress: 100
+      )
+    end
+
+    # Marks the run as failed.
+    #
+    # @param error [String, Exception] The error message or exception
+    # @return [void]
+    def fail!(error)
+      message = error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
+      update!(status: :failed, error_message: message)
+    end
+
+    # Returns the human-readable label for the current step.
+    #
+    # @return [String]
+    def current_step_label
+      STEPS.dig(current_step, :label) || current_step&.humanize || "Initializing"
+    end
+
+    # Returns elapsed time since the run started.
+    #
+    # @return [ActiveSupport::Duration, nil]
+    def elapsed_time
+      return nil unless running? || completed? || failed?
+
+      (completed? || failed? ? updated_at : Time.current) - created_at
+    end
+
+    # Formats elapsed time for display.
+    #
+    # @return [String]
+    def elapsed_time_formatted
+      seconds = elapsed_time&.to_i || 0
+      if seconds < 60
+        "#{seconds}s"
+      elsif seconds < 3600
+        "#{seconds / 60}m #{seconds % 60}s"
+      else
+        "#{seconds / 3600}h #{(seconds % 3600) / 60}m"
+      end
+    end
+
+    # Returns a hash for JSON API response.
+    #
+    # @return [Hash]
+    def as_json(options = {})
+      {
+        id: id,
+        status: status,
+        current_step: current_step,
+        current_step_label: current_step_label,
+        progress: progress,
+        examples_processed: examples_processed,
+        total_examples: total_examples,
+        elapsed_time: elapsed_time_formatted,
+        error_message: error_message,
+        prompt_id: prompt_id,
+        prompt_name: prompt_name
+      }
+    end
+  end
+end

data/app/models/leva/prompt.rb

@@ -26,5 +26,15 @@ module Leva
       self.version ||= 0
       self.version += 1
     end
+
+    # @return [Boolean] true if this prompt was generated by an optimizer
+    def optimized?
+      metadata&.dig("optimization").present?
+    end
+
+    # @return [String, nil] the optimizer used (e.g., "bootstrap", "mipro")
+    def optimizer_name
+      metadata&.dig("optimization", "optimizer")
+    end
   end
 end

data/app/services/leva/class_loader.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Leva
+  # Service for loading evaluator and runner classes from the application.
+  #
+  # This service dynamically loads classes from the app/evals and app/runners
+  # directories that inherit from their respective base classes.
+  class ClassLoader
+    # Loads all evaluator classes that inherit from Leva::BaseEval
+    #
+    # @return [Array<Class>] An array of evaluator classes
+    def self.evaluators
+      load_classes_from_directory("app/evals", Leva::BaseEval)
+    end
+
+    # Loads all runner classes that inherit from Leva::BaseRun
+    #
+    # @return [Array<Class>] An array of runner classes
+    def self.runners
+      load_classes_from_directory("app/runners", Leva::BaseRun)
+    end
+
+    # Loads classes from a specified directory that inherit from a given base class
+    #
+    # @param directory [String] The directory path to load classes from
+    # @param base_class [Class] The base class that loaded classes should inherit from
+    # @return [Array<Class>] An array of loaded classes
+    def self.load_classes_from_directory(directory, base_class)
+      classes = Dir[Rails.root.join(directory, "*.rb")].map do |file|
+        File.basename(file, ".rb").camelize.constantize
+      end.select { |klass| klass < base_class }
+      classes.empty? ? [] : classes
+    end
+
+    private_class_method :load_classes_from_directory
+  end
+end

data/app/services/leva/dataset_converter.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Leva
+  # Converts Leva datasets to DSPy example format.
+  #
+  # This service transforms DatasetRecord objects into DSPy::Example objects
+  # suitable for use with DSPy optimizers and predictors.
+  #
+  # @example Convert a dataset to DSPy examples
+  #   converter = Leva::DatasetConverter.new(dataset)
+  #   examples = converter.to_dspy_examples
+  #
+  # @example Split dataset for training
+  #   converter = Leva::DatasetConverter.new(dataset)
+  #   splits = converter.split(train_ratio: 0.6, val_ratio: 0.2)
+  #   # => { train: [...], val: [...], test: [...] }
+  class DatasetConverter
+    # @param dataset [Leva::Dataset] The dataset to convert
+    def initialize(dataset)
+      @dataset = dataset
+    end
+
+    # Converts all dataset records to DSPy example format.
+    #
+    # @return [Array<Hash>] Array of example hashes with :input and :expected keys
+    def to_dspy_examples
+      @dataset.dataset_records.includes(:recordable).map do |record|
+        next unless record.recordable
+
+        {
+          input: record.recordable.to_llm_context,
+          expected: { output: record.recordable.ground_truth }
+        }
+      end.compact
+    end
+
+    # Splits the dataset into train, validation, and test sets.
+    #
+    # @param train_ratio [Float] Proportion of data for training (default: 0.6)
+    # @param val_ratio [Float] Proportion of data for validation (default: 0.2)
+    # @param seed [Integer, nil] Random seed for reproducibility
+    # @return [Hash] Hash with :train, :val, and :test arrays
+    def split(train_ratio: 0.6, val_ratio: 0.2, seed: nil)
+      examples = to_dspy_examples
+      examples = seed ? examples.shuffle(random: Random.new(seed)) : examples.shuffle
+
+      train_size = (examples.size * train_ratio).to_i
+      val_size = (examples.size * val_ratio).to_i
+
+      {
+        train: examples[0...train_size],
+        val: examples[train_size...(train_size + val_size)],
+        test: examples[(train_size + val_size)..]
+      }
+    end
+
+    # Returns the count of valid records in the dataset.
+    #
+    # @return [Integer] Number of records with valid recordable objects
+    def valid_record_count
+      to_dspy_examples.size
+    end
+  end
+end