leva 0.2.0 → 0.3.1

data/app/models/leva/optimization_run.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Leva
+  # Tracks the progress and status of prompt optimization runs.
+  #
+  # @example Create and track an optimization run
+  #   run = OptimizationRun.create!(
+  #     dataset: dataset,
+  #     prompt_name: "My Optimized Prompt",
+  #     mode: "light"
+  #   )
+  #   run.start!
+  #   run.update_progress(step: "bootstrapping", progress: 50, examples_processed: 5)
+  #   run.complete!(prompt)
+  class OptimizationRun < ApplicationRecord
+    self.table_name = "leva_optimization_runs"
+
+    belongs_to :dataset
+    belongs_to :prompt, optional: true
+
+    enum :status, {
+      pending: "pending",
+      running: "running",
+      completed: "completed",
+      failed: "failed"
+    }, default: :pending
+
+    validates :prompt_name, presence: true, length: { maximum: 255 }
+    validates :mode, presence: true, inclusion: { in: %w[light medium heavy] }
+    validates :model, presence: true
+    validates :optimizer, inclusion: { in: PromptOptimizer::OPTIMIZERS.keys.map(&:to_s) }
+    validates :progress, numericality: { in: 0..100 }
+
+    # Defined optimization steps for display
+    STEPS = {
+      "validating" => { label: "Validating dataset", icon: "check" },
+      "splitting_data" => { label: "Splitting data", icon: "scissors" },
+      "generating_signature" => { label: "Generating signature", icon: "code" },
+      "bootstrapping" => { label: "Bootstrapping examples", icon: "zap" },
+      "evaluating" => { label: "Evaluating results", icon: "bar-chart" },
+      "building_result" => { label: "Building prompt", icon: "package" },
+      "complete" => { label: "Complete", icon: "check-circle" }
+    }.freeze
+
+    # Marks the run as started.
+    #
+    # @return [void]
+    def start!
+      update!(status: :running, current_step: "validating", progress: 0)
+    end
+
+    # Updates the progress of the optimization run.
+    #
+    # @param step [String] Current step name
+    # @param progress [Integer] Progress percentage (0-100)
+    # @param examples_processed [Integer, nil] Number of examples processed
+    # @param total [Integer, nil] Total examples to process
+    # @return [void]
+    def update_progress(step:, progress:, examples_processed: nil, total: nil)
+      attrs = { current_step: step, progress: progress }
+      attrs[:examples_processed] = examples_processed if examples_processed
+      attrs[:total_examples] = total if total
+      update!(attrs)
+    end
+
+    # Marks the run as completed with the created prompt.
+    #
+    # @param created_prompt [Leva::Prompt] The optimized prompt
+    # @return [void]
+    def complete!(created_prompt)
+      update!(
+        status: :completed,
+        prompt: created_prompt,
+        current_step: "complete",
+        progress: 100
+      )
+    end
+
+    # Marks the run as failed.
+    #
+    # @param error [String, Exception] The error message or exception
+    # @return [void]
+    def fail!(error)
+      message = error.is_a?(Exception) ? "#{error.class}: #{error.message}" : error.to_s
+      update!(status: :failed, error_message: message)
+    end
+
+    # Returns the human-readable label for the current step.
+    #
+    # @return [String]
+    def current_step_label
+      STEPS.dig(current_step, :label) || current_step&.humanize || "Initializing"
+    end
+
+    # Returns elapsed time since the run started.
+    #
+    # @return [ActiveSupport::Duration, nil]
+    def elapsed_time
+      return nil unless running? || completed? || failed?
+
+      (completed? || failed? ? updated_at : Time.current) - created_at
+    end
+
+    # Formats elapsed time for display.
+    #
+    # @return [String]
+    def elapsed_time_formatted
+      seconds = elapsed_time&.to_i || 0
+      if seconds < 60
+        "#{seconds}s"
+      elsif seconds < 3600
+        "#{seconds / 60}m #{seconds % 60}s"
+      else
+        "#{seconds / 3600}h #{(seconds % 3600) / 60}m"
+      end
+    end
+
+    # Returns a hash for JSON API response.
+    #
+    # @return [Hash]
+    def as_json(options = {})
+      {
+        id: id,
+        status: status,
+        current_step: current_step,
+        current_step_label: current_step_label,
+        progress: progress,
+        examples_processed: examples_processed,
+        total_examples: total_examples,
+        elapsed_time: elapsed_time_formatted,
+        error_message: error_message,
+        prompt_id: prompt_id,
+        prompt_name: prompt_name
+      }
+    end
+  end
+end

data/app/models/leva/prompt.rb

@@ -26,5 +26,15 @@ module Leva
       self.version ||= 0
       self.version += 1
     end
+
+    # @return [Boolean] true if this prompt was generated by an optimizer
+    def optimized?
+      metadata&.dig("optimization").present?
+    end
+
+    # @return [String, nil] the optimizer used (e.g., "bootstrap", "mipro")
+    def optimizer_name
+      metadata&.dig("optimization", "optimizer")
+    end
   end
 end

data/app/services/leva/class_loader.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Leva
+  # Service for loading evaluator and runner classes from the application.
+  #
+  # This service dynamically loads classes from the app/evals and app/runners
+  # directories that inherit from their respective base classes.
+  class ClassLoader
+    # Loads all evaluator classes that inherit from Leva::BaseEval
+    #
+    # @return [Array<Class>] An array of evaluator classes
+    def self.evaluators
+      load_classes_from_directory("app/evals", Leva::BaseEval)
+    end
+
+    # Loads all runner classes that inherit from Leva::BaseRun
+    #
+    # @return [Array<Class>] An array of runner classes
+    def self.runners
+      load_classes_from_directory("app/runners", Leva::BaseRun)
+    end
+
+    # Loads classes from a specified directory that inherit from a given base class
+    #
+    # @param directory [String] The directory path to load classes from
+    # @param base_class [Class] The base class that loaded classes should inherit from
+    # @return [Array<Class>] An array of loaded classes
+    def self.load_classes_from_directory(directory, base_class)
+      classes = Dir[Rails.root.join(directory, "*.rb")].map do |file|
+        File.basename(file, ".rb").camelize.constantize
+      end.select { |klass| klass < base_class }
+      classes.empty? ? [] : classes
+    end
+
+    private_class_method :load_classes_from_directory
+  end
+end

data/app/services/leva/dataset_converter.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Leva
+  # Converts Leva datasets to DSPy example format.
+  #
+  # This service transforms DatasetRecord objects into DSPy::Example objects
+  # suitable for use with DSPy optimizers and predictors.
+  #
+  # @example Convert a dataset to DSPy examples
+  #   converter = Leva::DatasetConverter.new(dataset)
+  #   examples = converter.to_dspy_examples
+  #
+  # @example Split dataset for training
+  #   converter = Leva::DatasetConverter.new(dataset)
+  #   splits = converter.split(train_ratio: 0.6, val_ratio: 0.2)
+  #   # => { train: [...], val: [...], test: [...] }
+  class DatasetConverter
+    # @param dataset [Leva::Dataset] The dataset to convert
+    def initialize(dataset)
+      @dataset = dataset
+    end
+
+    # Converts all dataset records to DSPy example format.
+    #
+    # @return [Array<Hash>] Array of example hashes with :input and :expected keys
+    def to_dspy_examples
+      @dataset.dataset_records.includes(:recordable).map do |record|
+        next unless record.recordable
+
+        {
+          input: record.recordable.to_llm_context,
+          expected: { output: record.recordable.ground_truth }
+        }
+      end.compact
+    end
+
+    # Splits the dataset into train, validation, and test sets.
+    #
+    # @param train_ratio [Float] Proportion of data for training (default: 0.6)
+    # @param val_ratio [Float] Proportion of data for validation (default: 0.2)
+    # @param seed [Integer, nil] Random seed for reproducibility
+    # @return [Hash] Hash with :train, :val, and :test arrays
+    def split(train_ratio: 0.6, val_ratio: 0.2, seed: nil)
+      examples = to_dspy_examples
+      examples = seed ? examples.shuffle(random: Random.new(seed)) : examples.shuffle
+
+      train_size = (examples.size * train_ratio).to_i
+      val_size = (examples.size * val_ratio).to_i
+
+      {
+        train: examples[0...train_size],
+        val: examples[train_size...(train_size + val_size)],
+        test: examples[(train_size + val_size)..]
+      }
+    end
+
+    # Returns the count of valid records in the dataset.
+    #
+    # @return [Integer] Number of records with valid recordable objects
+    def valid_record_count
+      to_dspy_examples.size
+    end
+  end
+end

data/app/services/leva/optimizers/base.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # Base class for optimization strategies.
+    #
+    # Each optimizer implements a different approach to finding optimal
+    # prompt instructions and few-shot examples.
+    #
+    # @abstract Subclass and override {#compile} to implement a strategy
+    class Base
+      # @return [String] The model identifier
+      attr_reader :model
+
+      # @return [Proc] The evaluation metric
+      attr_reader :metric
+
+      # @return [Proc, nil] Progress callback
+      attr_reader :progress_callback
+
+      # @return [Symbol] The optimization mode
+      attr_reader :mode
+
+      # @param model [String] The model to use for optimization
+      # @param metric [Proc] The evaluation metric
+      # @param mode [Symbol] Optimization intensity (:light, :medium, :heavy)
+      # @param progress_callback [Proc, nil] Callback for progress updates
+      def initialize(model:, metric:, mode:, progress_callback: nil)
+        @model = model
+        @metric = metric
+        @mode = mode
+        @progress_callback = progress_callback
+        @last_progress = nil
+      end
+
+      # Runs the optimization and returns results.
+      #
+      # @param splits [Hash] The train/val/test splits
+      # @param signature [Class] The generated DSPy signature class
+      # @return [Hash] Hash with :instruction, :few_shot_examples, :score
+      # @raise [Leva::OptimizationError] If optimization fails
+      def optimize(splits, signature)
+        train_examples = splits[:train]
+        val_examples = splits[:val]
+
+        report_progress(step: step_name, progress: 30, examples_processed: 0, total: train_examples.size)
+
+        predictor = DSPy::Predict.new(signature)
+        predictor.config.lm = create_lm
+
+        result = compile(predictor, train_examples, val_examples, signature)
+
+        report_progress(step: "evaluating", progress: 85)
+
+        instruction = result[:instruction_override] || extract_instruction(result[:optimized], signature)
+        score = evaluate(result[:optimized] || predictor, val_examples)
+
+        report_progress(step: "building_result", progress: 95)
+
+        {
+          instruction: instruction,
+          few_shot_examples: result[:few_shot_examples] || [],
+          score: score
+        }
+      rescue StandardError => e
+        Rails.logger.error "[Leva::Optimizers::#{self.class.name.demodulize}] Optimization failed: #{e.message}"
+        Rails.logger.error e.backtrace.first(5).join("\n")
+        raise Leva::OptimizationError, "#{optimizer_name} optimization failed: #{e.message}"
+      end
+
+      # The name used in progress reporting.
+      # @return [String]
+      def step_name
+        raise NotImplementedError, "Subclasses must implement #step_name"
+      end
+
+      # Human-readable optimizer name.
+      # @return [String]
+      def optimizer_name
+        raise NotImplementedError, "Subclasses must implement #optimizer_name"
+      end
+
+      # @return [Symbol] The optimizer type symbol
+      def optimizer_type
+        raise NotImplementedError, "Subclasses must implement #optimizer_type"
+      end
+
+      protected
+
+      # Performs the actual optimization logic.
+      #
+      # @param predictor [DSPy::Predict] The base predictor
+      # @param train_examples [Array<Hash>] Training examples
+      # @param val_examples [Array<Hash>] Validation examples
+      # @param signature [Class] The signature class
+      # @return [Hash] Hash with :optimized (predictor), :few_shot_examples, :instruction_override (optional)
+      def compile(predictor, train_examples, val_examples, signature)
+        raise NotImplementedError, "Subclasses must implement #compile"
+      end
+
+      # Converts examples to DSPy::Example format.
+      #
+      # @param examples [Array<Hash>] Examples with :input and :expected keys
+      # @param signature [Class] The DSPy signature class
+      # @return [Array<DSPy::Example>]
+      def to_dspy_examples(examples, signature)
+        examples.map do |ex|
+          DSPy::Example.new(
+            signature_class: signature,
+            input: ex[:input],
+            expected: ex[:expected]
+          )
+        end
+      end
+
+      # Creates an LM instance for this optimizer.
+      # Prepends ruby_llm/ prefix for DSPy adapter.
+      # RubyLLM handles API keys from its configuration.
+      #
+      # @return [DSPy::LM]
+      def create_lm
+        DSPy::LM.new("ruby_llm/#{model}")
+      end
+
+      # Reports progress to the callback if provided.
+      # Throttles updates to only report when progress changes by 5% or more.
+      def report_progress(step:, progress:, examples_processed: nil, total: nil)
+        return unless progress_callback
+
+        # Skip if progress hasn't changed by at least 5%
+        return if @last_progress && (progress - @last_progress).abs < 5
+
+        @last_progress = progress
+        progress_callback.call(
+          step: step,
+          progress: progress,
+          examples_processed: examples_processed,
+          total: total
+        )
+      end
+
+      private
+
+      # Extracts instruction from an optimized predictor.
+      def extract_instruction(optimized, signature)
+        return signature.description unless optimized
+
+        if optimized.respond_to?(:instruction)
+          optimized.instruction
+        elsif optimized.respond_to?(:signature) && optimized.signature.respond_to?(:instructions)
+          optimized.signature.instructions
+        else
+          signature.description
+        end
+      end
+
+      # Evaluates predictor on validation examples.
+      # Handles both Hash examples and DSPy::Example objects.
+      def evaluate(predictor, val_examples)
+        return 0.0 if val_examples.empty?
+
+        correct = val_examples.count do |example|
+          # Handle both Hash and DSPy::Example
+          if example.is_a?(Hash)
+            input = example[:input]
+            expected_output = example.dig(:expected, :output)
+          else
+            # DSPy::Example has input_values/expected_values methods
+            input = example.input_values
+            expected_output = example.expected_values[:output]
+          end
+
+          prediction = predictor.call(**input)
+          actual = prediction.output.to_s.strip.downcase
+          expected = expected_output.to_s.strip.downcase
+          actual == expected
+        end
+
+        correct.to_f / val_examples.size
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/bootstrap.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # Bootstrap optimization strategy.
+    #
+    # Uses simple few-shot bootstrapping to find the best examples
+    # that improve model performance. Fast and has no extra dependencies.
+    #
+    # @example
+    #   optimizer = Leva::Optimizers::Bootstrap.new(
+    #     model: "gemini-2.5-flash",
+    #     metric: my_metric,
+    #     mode: :medium
+    #   )
+    #   result = optimizer.optimize(splits, signature)
+    class Bootstrap < Base
+      MODES = {
+        light: { trials: 5 },
+        medium: { trials: 15 },
+        heavy: { trials: 30 }
+      }.freeze
+
+      def step_name
+        "bootstrapping"
+      end
+
+      def optimizer_name
+        "Bootstrap"
+      end
+
+      def optimizer_type
+        :bootstrap
+      end
+
+      protected
+
+      def compile(predictor, train_examples, _val_examples, signature)
+        best_examples = bootstrap_few_shot_examples(predictor, train_examples)
+        instruction = generate_optimized_instruction(best_examples, signature)
+
+        {
+          optimized: nil,
+          few_shot_examples: best_examples,
+          instruction_override: instruction
+        }
+      end
+
+      private
+
+      # Bootstraps few-shot examples by evaluating which examples help the model most.
+      def bootstrap_few_shot_examples(predictor, examples)
+        max_examples = MODES[mode][:trials].clamp(3, 8)
+
+        scored_examples = examples.each_with_index.map do |example, index|
+          prediction = predictor.call(**example[:input])
+          actual_output = prediction.output.to_s.strip.downcase
+          expected_output = example.dig(:expected, :output).to_s.strip.downcase
+
+          score = actual_output == expected_output ? 1.0 : 0.0
+
+          # Progress: 30% + (index / total * 50%)
+          progress = 30 + ((index + 1).to_f / examples.size * 50).to_i
+          report_progress(
+            step: step_name,
+            progress: progress,
+            examples_processed: index + 1,
+            total: examples.size
+          )
+
+          { example: example, score: score }
+        end
+
+        scored_examples.sort_by { |e| -e[:score] }.take(max_examples).map { |e| e[:example] }
+      end
+
+      # Generates instruction based on output patterns.
+      def generate_optimized_instruction(examples, signature)
+        return signature.description if examples.empty?
+
+        outputs = examples.map { |e| e.dig(:expected, :output) }.compact.uniq
+
+        if outputs.size <= 5
+          # Classification task
+          "#{signature.description} Respond with one of: #{outputs.join(', ')}."
+        else
+          signature.description
+        end
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/gepa_optimizer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # GEPA (Genetic-Pareto) optimization strategy.
+    #
+    # Uses reflective prompt evolution with genetic algorithms
+    # and Pareto optimization for best quality results.
+    #
+    # Requires the dspy-gepa gem.
+    class GepaOptimizer < Base
+      def step_name
+        "gepa_optimizing"
+      end
+
+      def optimizer_name
+        "GEPA"
+      end
+
+      def optimizer_type
+        :gepa
+      end
+
+      protected
+
+      def compile(predictor, train_examples, val_examples, signature)
+        lm = create_lm
+        predictor.config.lm = lm
+
+        # Build config based on mode intensity
+        gepa_config = case mode
+        when :light
+                        { max_metric_calls: 16, minibatch_size: 2 }
+        when :medium
+                        { max_metric_calls: 32, minibatch_size: 2 }
+        when :heavy
+                        { max_metric_calls: 64, minibatch_size: 4 }
+        else
+                        { max_metric_calls: 16, minibatch_size: 2 }
+        end
+
+        gepa = DSPy::Teleprompt::GEPA.new(
+          metric: metric,
+          reflection_lm: DSPy::ReflectionLM.new("ruby_llm/#{model}"),
+          config: gepa_config
+        )
+
+        trainset = to_dspy_examples(train_examples, signature)
+        valset = to_dspy_examples(val_examples, signature)
+
+        report_progress(step: "gepa_compiling", progress: 50)
+
+        result = gepa.compile(predictor, trainset: trainset, valset: valset)
+
+        { optimized: result.optimized_program, few_shot_examples: [] }
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/miprov2_optimizer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # MIPROv2 optimization strategy.
+    #
+    # Uses Bayesian optimization with Gaussian Processes
+    # for efficient prompt search.
+    #
+    # Requires the dspy-miprov2 gem.
+    class Miprov2Optimizer < Base
+      def step_name
+        "miprov2_optimizing"
+      end
+
+      def optimizer_name
+        "MIPROv2"
+      end
+
+      def optimizer_type
+        :miprov2
+      end
+
+      protected
+
+      def compile(predictor, train_examples, val_examples, signature)
+        predictor.config.lm = create_lm
+
+        # Use AutoMode helpers for preset configurations
+        mipro = case mode
+        when :light
+                  DSPy::Teleprompt::MIPROv2::AutoMode.light(metric: metric)
+        when :medium
+                  DSPy::Teleprompt::MIPROv2::AutoMode.medium(metric: metric)
+        when :heavy
+                  DSPy::Teleprompt::MIPROv2::AutoMode.heavy(metric: metric)
+        else
+                  DSPy::Teleprompt::MIPROv2::AutoMode.light(metric: metric)
+        end
+
+        trainset = to_dspy_examples(train_examples, signature)
+        valset = to_dspy_examples(val_examples, signature)
+
+        report_progress(step: "miprov2_compiling", progress: 50)
+
+        result = mipro.compile(predictor, trainset: trainset, valset: valset)
+
+        { optimized: result.optimized_program, few_shot_examples: [] }
+      end
+    end
+  end
+end