leva 0.2.0 → 0.3.1

data/app/services/leva/prompt_optimizer.rb

@@ -0,0 +1,305 @@
+# frozen_string_literal: true
+
+module Leva
+  # Optimizes prompts using DSPy.rb optimizers.
+  #
+  # This service coordinates the optimization process, delegating
+  # the actual optimization work to strategy classes.
+  #
+  # @example Optimize a prompt for a dataset
+  #   optimizer = Leva::PromptOptimizer.new(dataset: dataset, mode: :medium)
+  #   result = optimizer.optimize
+  #   # => { system_prompt: "...", user_prompt: "...", metadata: {...} }
+  #
+  # @example With GEPA optimizer
+  #   optimizer = Leva::PromptOptimizer.new(dataset: dataset, optimizer: :gepa, mode: :medium)
+  #   result = optimizer.optimize
+  class PromptOptimizer
+    # Minimum number of examples required for optimization
+    MINIMUM_EXAMPLES = 10
+
+    # Available optimizers with their strategy classes
+    OPTIMIZERS = {
+      bootstrap: {
+        name: "Bootstrap",
+        strategy_class: Leva::Optimizers::Bootstrap,
+        gem: nil,
+        description: "Fast and simple. Automatically selects optimal few-shot examples from your dataset. " \
+                     "Best for quick iteration and when you have limited data (10-50 examples). " \
+                     "Does not modify instructions, only adds demonstrations."
+      },
+      gepa: {
+        name: "GEPA",
+        strategy_class: Leva::Optimizers::GepaOptimizer,
+        gem: "dspy-gepa",
+        description: "State-of-the-art optimizer using reflective prompt evolution. Uses LLM reflection " \
+                     "to identify what works and propose improvements. Outperforms MIPROv2 by 10-14% " \
+                     "while being more sample efficient. Best choice for maximum quality."
+      },
+      miprov2: {
+        name: "MIPROv2",
+        strategy_class: Leva::Optimizers::Miprov2Optimizer,
+        gem: "dspy-miprov2",
+        description: "Uses Bayesian optimization to search for optimal instructions and few-shot examples. " \
+                     "Good for larger datasets (200+ examples). More computationally demanding but thorough. " \
+                     "Can overfit on small datasets."
+      }
+    }.freeze
+
+    # Default optimizer
+    DEFAULT_OPTIMIZER = :bootstrap
+
+    # Optimization modes with their approximate durations
+    MODES = {
+      light: { description: "Fast optimization (~5 min)", trials: 5 },
+      medium: { description: "Balanced optimization (~15 min)", trials: 15 },
+      heavy: { description: "Thorough optimization (~30 min)", trials: 30 }
+    }.freeze
+
+    # Default model if none specified (fast and cheap)
+    DEFAULT_MODEL = "gemini-2.5-flash"
+
+    # Returns available models from RubyLLM.
+    # Results are cached for 5 minutes to avoid repeated expensive calls.
+    #
+    # @return [Array<RubyLLM::Model>] All available chat models
+    def self.available_models
+      Rails.cache.fetch("leva/available_models", expires_in: 5.minutes) do
+        RubyLLM.models.chat_models
+      end
+    end
+
+    # Finds a model by ID.
+    #
+    # @param model_id [String] The model ID to find
+    # @return [RubyLLM::Model, nil] The model or nil if not found
+    def self.find_model(model_id)
+      RubyLLM.models.find(model_id)
+    rescue RubyLLM::ModelNotFoundError
+      nil
+    end
+
+    # @return [Leva::Dataset] The dataset being optimized
+    attr_reader :dataset
+
+    # @return [Symbol] The optimization mode (:light, :medium, :heavy)
+    attr_reader :mode
+
+    # @return [String] The model to use for optimization
+    attr_reader :model
+
+    # @return [Symbol] The optimizer to use (:bootstrap, :gepa, :miprov2)
+    attr_reader :optimizer
+
+    # @param dataset [Leva::Dataset] The dataset to optimize for
+    # @param metric [Proc, nil] Custom evaluation metric (default: exact string match)
+    # @param mode [Symbol] Optimization intensity (:light, :medium, :heavy)
+    # @param model [String, nil] The model to use (default: DEFAULT_MODEL)
+    # @param optimizer [Symbol, String] The optimizer to use (default: :bootstrap)
+    # @param progress_callback [Proc, nil] Callback for progress updates
+    def initialize(dataset:, metric: nil, mode: :light, model: nil, optimizer: nil, progress_callback: nil)
+      @dataset = dataset
+      @metric = metric || default_metric
+      @mode = mode.to_sym
+      @model = model.presence || DEFAULT_MODEL
+      @optimizer = (optimizer.presence || DEFAULT_OPTIMIZER).to_sym
+      @progress_callback = progress_callback
+      @last_progress = nil
+    end
+
+    # Runs the optimization process.
+    #
+    # @return [Hash] Hash containing :system_prompt, :user_prompt, and :metadata
+    # @raise [Leva::InsufficientDataError] If dataset has too few records
+    # @raise [Leva::DspyConfigurationError] If DSPy is not configured
+    def optimize
+      report_progress(step: "validating", progress: 0)
+      validate_dataset!
+      validate_dspy_configuration!
+      validate_optimizer!
+
+      report_progress(step: "splitting_data", progress: 10)
+      splits = DatasetConverter.new(@dataset).split
+
+      report_progress(step: "generating_signature", progress: 20)
+      signature = SignatureGenerator.new(@dataset).generate
+
+      # Delegate to optimizer strategy
+      strategy = build_optimizer_strategy
+      result = strategy.optimize(splits, signature)
+
+      report_progress(step: "complete", progress: 100)
+
+      build_final_result(result, splits, strategy.optimizer_type)
+    end
+
+    # Checks if the dataset is ready for optimization.
+    #
+    # @return [Boolean] True if the dataset can be optimized
+    def can_optimize?
+      @dataset.dataset_records.count >= MINIMUM_EXAMPLES
+    end
+
+    # Returns the number of additional records needed for optimization.
+    #
+    # @return [Integer] Number of records still needed (0 if ready)
+    def records_needed
+      [ MINIMUM_EXAMPLES - @dataset.dataset_records.count, 0 ].max
+    end
+
+    # Checks if a specific optimizer is available.
+    #
+    # @param optimizer_type [Symbol] The optimizer to check
+    # @return [Boolean] True if the optimizer is available
+    def self.optimizer_available?(optimizer_type)
+      optimizer_type = optimizer_type.to_sym
+      return true if optimizer_type == :bootstrap
+
+      case optimizer_type
+      when :gepa
+        !!defined?(DSPy::Teleprompt::GEPA)
+      when :miprov2
+        !!defined?(DSPy::Teleprompt::MIPROv2)
+      else
+        false
+      end
+    end
+
+    private
+
+    # Builds the optimizer strategy instance.
+    #
+    # @return [Leva::Optimizers::Base] The optimizer strategy
+    def build_optimizer_strategy
+      config = OPTIMIZERS[@optimizer]
+      config[:strategy_class].new(
+        model: @model,
+        metric: @metric,
+        mode: @mode,
+        progress_callback: @progress_callback
+      )
+    end
+
+    # Builds the final result hash from optimization.
+    #
+    # @param result [Hash] The optimizer result with :instruction, :few_shot_examples, :score
+    # @param splits [Hash] The data splits
+    # @param optimizer_type [Symbol] The optimizer that was used
+    # @return [Hash] The formatted result
+    def build_final_result(result, splits, optimizer_type)
+      sample_record = @dataset.dataset_records.first&.recordable
+      input_fields = sample_record&.to_llm_context&.keys || []
+
+      formatted_examples = result[:few_shot_examples].map do |ex|
+        { input: ex[:input], output: ex.dig(:expected, :output) }
+      end
+
+      {
+        system_prompt: result[:instruction],
+        user_prompt: build_user_prompt_template(input_fields),
+        metadata: {
+          optimization: {
+            score: result[:score],
+            mode: @mode.to_s,
+            optimizer: optimizer_type.to_s,
+            model: @model,
+            few_shot_examples: formatted_examples,
+            optimized_at: Time.current.iso8601,
+            dataset_size: @dataset.dataset_records.count,
+            train_size: splits[:train].size,
+            val_size: splits[:val].size,
+            test_size: splits[:test].size
+          }
+        }
+      }
+    end
+
+    # Reports progress to the callback if provided.
+    # Throttles updates to only report when progress changes by 5% or more.
+    #
+    # @param step [String] Current step name
+    # @param progress [Integer] Progress percentage (0-100)
+    # @param examples_processed [Integer, nil] Number of examples processed
+    # @param total [Integer, nil] Total examples to process
+    # @return [void]
+    def report_progress(step:, progress:, examples_processed: nil, total: nil)
+      return unless @progress_callback
+
+      # Skip if progress hasn't changed by at least 5%
+      return if @last_progress && (progress - @last_progress).abs < 5
+
+      @last_progress = progress
+      @progress_callback.call(
+        step: step,
+        progress: progress,
+        examples_processed: examples_processed,
+        total: total
+      )
+    end
+
+    # Validates that the dataset has enough records.
+    #
+    # @raise [Leva::InsufficientDataError] If dataset has too few records
+    def validate_dataset!
+      count = @dataset.dataset_records.count
+      return if count >= MINIMUM_EXAMPLES
+
+      raise InsufficientDataError,
+        "Dataset needs at least #{MINIMUM_EXAMPLES} records for optimization, has #{count}"
+    end
+
+    # Validates that DSPy is properly configured.
+    #
+    # @raise [Leva::DspyConfigurationError] If DSPy is not configured
+    def validate_dspy_configuration!
+      unless defined?(DSPy) && defined?(DSPy::Predict)
+        raise DspyConfigurationError, "DSPy is not installed. Add 'dspy' gem to your Gemfile."
+      end
+    end
+
+    # Validates that the selected optimizer is available.
+    #
+    # @raise [Leva::DspyConfigurationError] If optimizer is not available
+    def validate_optimizer!
+      return if @optimizer == :bootstrap
+      return if self.class.optimizer_available?(@optimizer)
+
+      gem_name = OPTIMIZERS.dig(@optimizer, :gem)
+      raise DspyConfigurationError, <<~MSG.strip
+        #{@optimizer.to_s.upcase} optimizer is not available. Install it:
+
+          gem 'dspy'
+          gem '#{gem_name}'
+
+        Or set DSPY_WITH_#{@optimizer.to_s.upcase}=1 before requiring dspy.
+      MSG
+    end
+
+    # Returns the default evaluation metric (case-insensitive exact match).
+    # Handles both Hash examples and DSPy::Example objects.
+    #
+    # @return [Proc] The default metric function
+    def default_metric
+      lambda do |example, prediction|
+        # Handle both Hash and DSPy::Example
+        expected_output = if example.is_a?(Hash)
+                            example.dig(:expected, :output)
+        else
+                            # DSPy::Example has expected_values method to get Hash
+                            example.expected_values[:output]
+        end
+        expected = expected_output.to_s.strip.downcase
+        actual = prediction.to_s.strip.downcase
+        expected == actual ? 1.0 : 0.0
+      end
+    end
+
+    # Builds the user prompt template with Liquid placeholders.
+    #
+    # @param input_fields [Array<Symbol>] The input field names
+    # @return [String] The user prompt template
+    def build_user_prompt_template(input_fields)
+      input_fields.map { |field| "{{ #{field} }}" }.join("\n\n")
+    end
+  end
+end

data/app/services/leva/signature_generator.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Leva
+  # Generates DSPy signatures from Leva dataset records.
+  #
+  # This service analyzes the structure of dataset records and generates
+  # a dynamic DSPy::Signature class that matches the input/output schema.
+  #
+  # @example Generate a signature from a dataset
+  #   generator = Leva::SignatureGenerator.new(dataset)
+  #   signature_class = generator.generate
+  #   predictor = DSPy::Predict.new(signature_class)
+  class SignatureGenerator
+    # @param dataset [Leva::Dataset] The dataset to analyze
+    # @param description [String, nil] Optional description for the signature
+    def initialize(dataset, description: nil)
+      @dataset = dataset
+      @description = description
+      @sample_record = dataset.dataset_records.first&.recordable
+    end
+
+    # Generates a DSPy::Signature class based on the dataset structure.
+    #
+    # @return [Class, nil] A dynamically generated DSPy::Signature subclass, or nil if no sample
+    def generate
+      return nil unless @sample_record
+
+      input_fields = extract_input_fields
+      output_type = infer_output_type(@sample_record.ground_truth)
+      description = @description || generate_description
+
+      build_signature_class(input_fields, output_type, description)
+    end
+
+    # Returns the input field names that will be used in the signature.
+    #
+    # @return [Array<Symbol>] Array of input field names
+    def input_field_names
+      return [] unless @sample_record
+
+      extract_input_fields.keys
+    end
+
+    private
+
+    # Extracts input fields from the sample record's LLM context.
+    #
+    # @return [Hash<Symbol, Class>] Map of field names to their inferred types
+    def extract_input_fields
+      context = @sample_record.to_llm_context
+      context.transform_values { |value| infer_type(value) }
+    end
+
+    # Infers the Ruby type for a given value.
+    #
+    # @param value [Object] The value to analyze
+    # @return [Class] The inferred type (String, Integer, Float, Array, or Hash)
+    def infer_type(value)
+      case value
+      when String then String
+      when Integer then Integer
+      when Float then Float
+      when Array then Array
+      when Hash then Hash
+      else String
+      end
+    end
+
+    # Infers the output type from ground truth.
+    #
+    # @param ground_truth [Object] The ground truth value to analyze
+    # @return [Symbol] The output type (:string, :array, or :hash)
+    def infer_output_type(ground_truth)
+      case ground_truth
+      when String then :string
+      when Array then :array
+      when Hash then :hash
+      else :string
+      end
+    end
+
+    # Generates a description for the signature based on the dataset.
+    #
+    # @return [String] A descriptive string for the signature
+    def generate_description
+      # Analyze ground truth values to determine task type
+      ground_truths = @dataset.dataset_records.limit(20).map { |r| r.recordable.ground_truth }.compact
+      unique_outputs = ground_truths.uniq
+
+      if unique_outputs.size <= 10
+        # Classification task - be explicit about output format
+        "Classify the input. Respond with ONLY one of these exact values, nothing else: #{unique_outputs.join(', ')}"
+      else
+        # Generation task
+        "Generate output for the given input from dataset: #{@dataset.name}"
+      end
+    end
+
+    # Builds the DSPy::Signature class dynamically.
+    #
+    # @param input_fields [Hash<Symbol, Class>] Input field definitions
+    # @param output_type [Symbol] The output type
+    # @param description [String] Description for the signature
+    # @return [Class] The generated DSPy::Signature subclass
+    # @raise [Leva::DspyConfigurationError] If DSPy is not available
+    def build_signature_class(input_fields, output_type, description)
+      unless defined?(DSPy::Signature)
+        raise DspyConfigurationError, "DSPy is required for signature generation"
+      end
+
+      captured_input_fields = input_fields
+      captured_description = description
+
+      Class.new(DSPy::Signature) do
+        description captured_description
+
+        input do
+          captured_input_fields.each do |name, _type|
+            const name, String
+          end
+        end
+
+        output do
+          const :output, String
+        end
+      end
+    end
+  end
+end

data/app/views/leva/datasets/show.html.erb

@@ -84,6 +84,9 @@
     <% end %>
   </section>
 
+  <%# Optimized Prompts Section - TODO: Enable when DSPy routes are added %>
+  <%# This feature is available in the PromptOptimizer service but UI routes are pending %>
+
   <%# Experiments Section %>
   <section>
     <div class="section-header">

data/app/views/leva/experiments/_experiment.html.erb

@@ -8,6 +8,9 @@
     else 'status-dot-pending'
   end
   run_count = experiment.runner_results.count
+
+  # Group evaluation results by evaluator_class to avoid N+1 queries
+  grouped_results = experiment.evaluation_results.group_by(&:evaluator_class)
 %>
 <tr class="experiment-row" onclick="window.location='<%= experiment_path(experiment) %>'">
   <td>
@@ -21,6 +24,9 @@
   <td>
     <span class="cell-dataset"><%= experiment.dataset&.name || '—' %></span>
   </td>
+  <td>
+    <span class="cell-model font-mono text-sm"><%= experiment.metadata&.dig("model") || '—' %></span>
+  </td>
   <td class="text-right text-nowrap">
     <span class="cell-timestamp"><%= time_ago_in_words(experiment.created_at) %></span>
   </td>
@@ -33,22 +39,15 @@
   <td class="text-right">
     <span class="cell-count"><%= run_count %></span>
   </td>
-  <% Leva::EvaluationResult.distinct.pluck(:evaluator_class).each do |evaluator_class| %>
+  <% @evaluator_classes.each do |evaluator_class| %>
     <td class="text-right">
-      <% results = experiment.evaluation_results.where(evaluator_class: evaluator_class) %>
+      <% results = grouped_results[evaluator_class] || [] %>
       <% if results.any? %>
         <%
           avg_score = (results.sum(&:score) / results.size.to_f)
           score_pct = (avg_score * 100).round
-          score_class = case avg_score
-            when 0...0.2 then 'score-bad'
-            when 0.2...0.4 then 'score-poor'
-            when 0.4...0.6 then 'score-fair'
-            when 0.6...0.8 then 'score-good'
-            else 'score-excellent'
-          end
         %>
-        <span class="score-pill <%= score_class %>"><%= score_pct %>%</span>
+        <span class="score-pill <%= score_class(avg_score) %>"><%= score_pct %>%</span>
       <% else %>
         <span class="score-empty">—</span>
       <% end %>

data/app/views/leva/experiments/_form.html.erb

@@ -50,6 +50,16 @@
                       class: "form-select" %>
       <p class="form-hint">The runner executes your model logic for each dataset record.</p>
     </div>
+
+    <div class="form-group" id="model-selection-group">
+      <label for="experiment_metadata_model" class="form-label">Model (for LLM runners)</label>
+      <select name="experiment[metadata][model]" id="experiment_metadata_model" class="form-select">
+        <% Leva::PromptOptimizer.available_models.each do |m| %>
+          <option value="<%= m.id %>" <%= 'selected' if @experiment.metadata&.dig("model") == m.id || (@experiment.metadata.blank? && m.id == "gemini-2.5-flash") %>><%= m.name %></option>
+        <% end %>
+      </select>
+      <p class="form-hint">The AI model to use when running LLM-based runners like SentimentLlmRun.</p>
+    </div>
   </div>
 
   <hr class="form-divider">

data/app/views/leva/experiments/index.html.erb

@@ -21,10 +21,11 @@
             <tr>
               <th>Experiment</th>
               <th style="width: 140px;">Dataset</th>
+              <th style="width: 140px;">Model</th>
               <th class="text-right" style="width: 90px;">Created</th>
               <th class="text-center" style="width: 90px;">Status</th>
               <th class="text-right" style="width: 60px;">Runs</th>
-              <% Leva::EvaluationResult.distinct.pluck(:evaluator_class).each do |evaluator_class| %>
+              <% @evaluator_classes.each do |evaluator_class| %>
                 <%
                   # Clean up evaluator name: "SentimentAccuracyEval" -> "Accuracy"
                   # Remove common prefixes/suffixes and module names

data/app/views/leva/experiments/show.html.erb

@@ -55,12 +55,27 @@
       </div>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Prompt</span>
-        <span class="exp-meta-value"><%= @experiment.prompt ? @experiment.prompt.name : '—' %></span>
+        <span class="exp-meta-value">
+          <% if @experiment.prompt %>
+            <%= @experiment.prompt.name %>
+            <% if @experiment.prompt.optimized? %>
+              <span class="badge badge-optimized" title="Generated by <%= @experiment.prompt.optimizer_name&.titleize || 'optimizer' %>">Optimized</span>
+            <% end %>
+          <% else %>
+            —
+          <% end %>
+        </span>
       </div>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Runner</span>
         <span class="exp-meta-value font-mono text-sm"><%= @experiment.runner_class&.demodulize || '—' %></span>
       </div>
+      <% if @experiment.metadata&.dig("model").present? %>
+        <div class="exp-meta-item">
+          <span class="exp-meta-label">Model</span>
+          <span class="exp-meta-value font-mono text-sm"><%= @experiment.metadata["model"] %></span>
+        </div>
+      <% end %>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Created</span>
         <span class="exp-meta-value"><%= time_ago_in_words(@experiment.created_at) %> ago</span>
@@ -79,13 +94,6 @@
           <%
             avg_score = (results.sum(&:score) / results.size.to_f).round(2)
             score_pct = (avg_score * 100).round
-            score_class = case avg_score
-              when 0...0.2 then 'score-bad'
-              when 0.2...0.4 then 'score-poor'
-              when 0.4...0.6 then 'score-fair'
-              when 0.6...0.8 then 'score-good'
-              else 'score-excellent'
-            end
             short_name = evaluator_class.demodulize
               .gsub(/Evaluator$/, '')
               .gsub(/Eval$/, '')
@@ -93,10 +101,10 @@
             short_name = short_name.presence || evaluator_class.demodulize.gsub(/Eval(uator)?$/, '')
           %>
           <div class="eval-summary-card" title="<%= results.size %> evaluations">
-            <span class="eval-summary-score <%= score_class %>"><%= score_pct %><span class="eval-summary-pct">%</span></span>
+            <span class="eval-summary-score <%= score_class(avg_score) %>"><%= score_pct %><span class="eval-summary-pct">%</span></span>
             <span class="eval-summary-name"><%= short_name %></span>
             <div class="eval-summary-bar">
-              <div class="eval-summary-bar-fill <%= score_class %>" style="width: <%= score_pct %>%"></div>
+              <div class="eval-summary-bar-fill <%= score_class(avg_score) %>" style="width: <%= score_pct %>%"></div>
             </div>
             <span class="eval-summary-count"><%= results.size %> runs</span>
           </div>
@@ -139,7 +147,7 @@
                     <span class="row-title"><%= runner_result.dataset_record.display_name %></span>
                   </td>
                   <td>
-                    <span class="prediction-badge"><%= truncate(runner_result.prediction.to_s.strip, length: 25) %></span>
+                    <span class="prediction-badge"><%= truncate(runner_result.parsed_predictions.first.to_s.presence || runner_result.prediction.to_s.strip, length: 25) %></span>
                   </td>
                   <td class="text-muted"><%= truncate(runner_result.ground_truth.to_s.strip.presence || '—', length: 25) %></td>
                   <% @experiment.evaluation_results.group_by(&:evaluator_class).keys.each do |evaluator_class| %>
@@ -147,16 +155,7 @@
                       <% eval_result = runner_result.evaluation_results.find_by(evaluator_class: evaluator_class) %>
                       <% if eval_result %>
                         <% score = eval_result.score %>
-                        <%
-                          score_class = case score
-                            when 0...0.2 then 'score-bad'
-                            when 0.2...0.4 then 'score-poor'
-                            when 0.4...0.6 then 'score-fair'
-                            when 0.6...0.8 then 'score-good'
-                            else 'score-excellent'
-                          end
-                        %>
-                        <span class="score-inline <%= score_class %>"><%= sprintf('%.2f', score) %></span>
+                        <span class="score-inline <%= score_class(score) %>"><%= sprintf('%.2f', score) %></span>
                       <% else %>
                         <span class="text-subtle">—</span>
                       <% end %>