leva 0.2.1 → 0.3.1

data/app/services/leva/optimizers/base.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # Base class for optimization strategies.
+    #
+    # Each optimizer implements a different approach to finding optimal
+    # prompt instructions and few-shot examples.
+    #
+    # @abstract Subclass and override {#compile} to implement a strategy
+    class Base
+      # @return [String] The model identifier
+      attr_reader :model
+
+      # @return [Proc] The evaluation metric
+      attr_reader :metric
+
+      # @return [Proc, nil] Progress callback
+      attr_reader :progress_callback
+
+      # @return [Symbol] The optimization mode
+      attr_reader :mode
+
+      # @param model [String] The model to use for optimization
+      # @param metric [Proc] The evaluation metric
+      # @param mode [Symbol] Optimization intensity (:light, :medium, :heavy)
+      # @param progress_callback [Proc, nil] Callback for progress updates
+      def initialize(model:, metric:, mode:, progress_callback: nil)
+        @model = model
+        @metric = metric
+        @mode = mode
+        @progress_callback = progress_callback
+        @last_progress = nil
+      end
+
+      # Runs the optimization and returns results.
+      #
+      # @param splits [Hash] The train/val/test splits
+      # @param signature [Class] The generated DSPy signature class
+      # @return [Hash] Hash with :instruction, :few_shot_examples, :score
+      # @raise [Leva::OptimizationError] If optimization fails
+      def optimize(splits, signature)
+        train_examples = splits[:train]
+        val_examples = splits[:val]
+
+        report_progress(step: step_name, progress: 30, examples_processed: 0, total: train_examples.size)
+
+        predictor = DSPy::Predict.new(signature)
+        predictor.config.lm = create_lm
+
+        result = compile(predictor, train_examples, val_examples, signature)
+
+        report_progress(step: "evaluating", progress: 85)
+
+        instruction = result[:instruction_override] || extract_instruction(result[:optimized], signature)
+        score = evaluate(result[:optimized] || predictor, val_examples)
+
+        report_progress(step: "building_result", progress: 95)
+
+        {
+          instruction: instruction,
+          few_shot_examples: result[:few_shot_examples] || [],
+          score: score
+        }
+      rescue StandardError => e
+        Rails.logger.error "[Leva::Optimizers::#{self.class.name.demodulize}] Optimization failed: #{e.message}"
+        Rails.logger.error e.backtrace.first(5).join("\n")
+        raise Leva::OptimizationError, "#{optimizer_name} optimization failed: #{e.message}"
+      end
+
+      # The name used in progress reporting.
+      # @return [String]
+      def step_name
+        raise NotImplementedError, "Subclasses must implement #step_name"
+      end
+
+      # Human-readable optimizer name.
+      # @return [String]
+      def optimizer_name
+        raise NotImplementedError, "Subclasses must implement #optimizer_name"
+      end
+
+      # @return [Symbol] The optimizer type symbol
+      def optimizer_type
+        raise NotImplementedError, "Subclasses must implement #optimizer_type"
+      end
+
+      protected
+
+      # Performs the actual optimization logic.
+      #
+      # @param predictor [DSPy::Predict] The base predictor
+      # @param train_examples [Array<Hash>] Training examples
+      # @param val_examples [Array<Hash>] Validation examples
+      # @param signature [Class] The signature class
+      # @return [Hash] Hash with :optimized (predictor), :few_shot_examples, :instruction_override (optional)
+      def compile(predictor, train_examples, val_examples, signature)
+        raise NotImplementedError, "Subclasses must implement #compile"
+      end
+
+      # Converts examples to DSPy::Example format.
+      #
+      # @param examples [Array<Hash>] Examples with :input and :expected keys
+      # @param signature [Class] The DSPy signature class
+      # @return [Array<DSPy::Example>]
+      def to_dspy_examples(examples, signature)
+        examples.map do |ex|
+          DSPy::Example.new(
+            signature_class: signature,
+            input: ex[:input],
+            expected: ex[:expected]
+          )
+        end
+      end
+
+      # Creates an LM instance for this optimizer.
+      # Prepends ruby_llm/ prefix for DSPy adapter.
+      # RubyLLM handles API keys from its configuration.
+      #
+      # @return [DSPy::LM]
+      def create_lm
+        DSPy::LM.new("ruby_llm/#{model}")
+      end
+
+      # Reports progress to the callback if provided.
+      # Throttles updates to only report when progress changes by 5% or more.
+      def report_progress(step:, progress:, examples_processed: nil, total: nil)
+        return unless progress_callback
+
+        # Skip if progress hasn't changed by at least 5%
+        return if @last_progress && (progress - @last_progress).abs < 5
+
+        @last_progress = progress
+        progress_callback.call(
+          step: step,
+          progress: progress,
+          examples_processed: examples_processed,
+          total: total
+        )
+      end
+
+      private
+
+      # Extracts instruction from an optimized predictor.
+      def extract_instruction(optimized, signature)
+        return signature.description unless optimized
+
+        if optimized.respond_to?(:instruction)
+          optimized.instruction
+        elsif optimized.respond_to?(:signature) && optimized.signature.respond_to?(:instructions)
+          optimized.signature.instructions
+        else
+          signature.description
+        end
+      end
+
+      # Evaluates predictor on validation examples.
+      # Handles both Hash examples and DSPy::Example objects.
+      def evaluate(predictor, val_examples)
+        return 0.0 if val_examples.empty?
+
+        correct = val_examples.count do |example|
+          # Handle both Hash and DSPy::Example
+          if example.is_a?(Hash)
+            input = example[:input]
+            expected_output = example.dig(:expected, :output)
+          else
+            # DSPy::Example has input_values/expected_values methods
+            input = example.input_values
+            expected_output = example.expected_values[:output]
+          end
+
+          prediction = predictor.call(**input)
+          actual = prediction.output.to_s.strip.downcase
+          expected = expected_output.to_s.strip.downcase
+          actual == expected
+        end
+
+        correct.to_f / val_examples.size
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/bootstrap.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # Bootstrap optimization strategy.
+    #
+    # Uses simple few-shot bootstrapping to find the best examples
+    # that improve model performance. Fast and has no extra dependencies.
+    #
+    # @example
+    #   optimizer = Leva::Optimizers::Bootstrap.new(
+    #     model: "gemini-2.5-flash",
+    #     metric: my_metric,
+    #     mode: :medium
+    #   )
+    #   result = optimizer.optimize(splits, signature)
+    class Bootstrap < Base
+      MODES = {
+        light: { trials: 5 },
+        medium: { trials: 15 },
+        heavy: { trials: 30 }
+      }.freeze
+
+      def step_name
+        "bootstrapping"
+      end
+
+      def optimizer_name
+        "Bootstrap"
+      end
+
+      def optimizer_type
+        :bootstrap
+      end
+
+      protected
+
+      def compile(predictor, train_examples, _val_examples, signature)
+        best_examples = bootstrap_few_shot_examples(predictor, train_examples)
+        instruction = generate_optimized_instruction(best_examples, signature)
+
+        {
+          optimized: nil,
+          few_shot_examples: best_examples,
+          instruction_override: instruction
+        }
+      end
+
+      private
+
+      # Bootstraps few-shot examples by evaluating which examples help the model most.
+      def bootstrap_few_shot_examples(predictor, examples)
+        max_examples = MODES[mode][:trials].clamp(3, 8)
+
+        scored_examples = examples.each_with_index.map do |example, index|
+          prediction = predictor.call(**example[:input])
+          actual_output = prediction.output.to_s.strip.downcase
+          expected_output = example.dig(:expected, :output).to_s.strip.downcase
+
+          score = actual_output == expected_output ? 1.0 : 0.0
+
+          # Progress: 30% + (index / total * 50%)
+          progress = 30 + ((index + 1).to_f / examples.size * 50).to_i
+          report_progress(
+            step: step_name,
+            progress: progress,
+            examples_processed: index + 1,
+            total: examples.size
+          )
+
+          { example: example, score: score }
+        end
+
+        scored_examples.sort_by { |e| -e[:score] }.take(max_examples).map { |e| e[:example] }
+      end
+
+      # Generates instruction based on output patterns.
+      def generate_optimized_instruction(examples, signature)
+        return signature.description if examples.empty?
+
+        outputs = examples.map { |e| e.dig(:expected, :output) }.compact.uniq
+
+        if outputs.size <= 5
+          # Classification task
+          "#{signature.description} Respond with one of: #{outputs.join(', ')}."
+        else
+          signature.description
+        end
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/gepa_optimizer.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # GEPA (Genetic-Pareto) optimization strategy.
+    #
+    # Uses reflective prompt evolution with genetic algorithms
+    # and Pareto optimization for best quality results.
+    #
+    # Requires the dspy-gepa gem.
+    class GepaOptimizer < Base
+      def step_name
+        "gepa_optimizing"
+      end
+
+      def optimizer_name
+        "GEPA"
+      end
+
+      def optimizer_type
+        :gepa
+      end
+
+      protected
+
+      def compile(predictor, train_examples, val_examples, signature)
+        lm = create_lm
+        predictor.config.lm = lm
+
+        # Build config based on mode intensity
+        gepa_config = case mode
+        when :light
+                        { max_metric_calls: 16, minibatch_size: 2 }
+        when :medium
+                        { max_metric_calls: 32, minibatch_size: 2 }
+        when :heavy
+                        { max_metric_calls: 64, minibatch_size: 4 }
+        else
+                        { max_metric_calls: 16, minibatch_size: 2 }
+        end
+
+        gepa = DSPy::Teleprompt::GEPA.new(
+          metric: metric,
+          reflection_lm: DSPy::ReflectionLM.new("ruby_llm/#{model}"),
+          config: gepa_config
+        )
+
+        trainset = to_dspy_examples(train_examples, signature)
+        valset = to_dspy_examples(val_examples, signature)
+
+        report_progress(step: "gepa_compiling", progress: 50)
+
+        result = gepa.compile(predictor, trainset: trainset, valset: valset)
+
+        { optimized: result.optimized_program, few_shot_examples: [] }
+      end
+    end
+  end
+end

data/app/services/leva/optimizers/miprov2_optimizer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Leva
+  module Optimizers
+    # MIPROv2 optimization strategy.
+    #
+    # Uses Bayesian optimization with Gaussian Processes
+    # for efficient prompt search.
+    #
+    # Requires the dspy-miprov2 gem.
+    class Miprov2Optimizer < Base
+      def step_name
+        "miprov2_optimizing"
+      end
+
+      def optimizer_name
+        "MIPROv2"
+      end
+
+      def optimizer_type
+        :miprov2
+      end
+
+      protected
+
+      def compile(predictor, train_examples, val_examples, signature)
+        predictor.config.lm = create_lm
+
+        # Use AutoMode helpers for preset configurations
+        mipro = case mode
+        when :light
+                  DSPy::Teleprompt::MIPROv2::AutoMode.light(metric: metric)
+        when :medium
+                  DSPy::Teleprompt::MIPROv2::AutoMode.medium(metric: metric)
+        when :heavy
+                  DSPy::Teleprompt::MIPROv2::AutoMode.heavy(metric: metric)
+        else
+                  DSPy::Teleprompt::MIPROv2::AutoMode.light(metric: metric)
+        end
+
+        trainset = to_dspy_examples(train_examples, signature)
+        valset = to_dspy_examples(val_examples, signature)
+
+        report_progress(step: "miprov2_compiling", progress: 50)
+
+        result = mipro.compile(predictor, trainset: trainset, valset: valset)
+
+        { optimized: result.optimized_program, few_shot_examples: [] }
+      end
+    end
+  end
+end

data/app/services/leva/prompt_optimizer.rb

@@ -0,0 +1,305 @@
+# frozen_string_literal: true
+
+module Leva
+  # Optimizes prompts using DSPy.rb optimizers.
+  #
+  # This service coordinates the optimization process, delegating
+  # the actual optimization work to strategy classes.
+  #
+  # @example Optimize a prompt for a dataset
+  #   optimizer = Leva::PromptOptimizer.new(dataset: dataset, mode: :medium)
+  #   result = optimizer.optimize
+  #   # => { system_prompt: "...", user_prompt: "...", metadata: {...} }
+  #
+  # @example With GEPA optimizer
+  #   optimizer = Leva::PromptOptimizer.new(dataset: dataset, optimizer: :gepa, mode: :medium)
+  #   result = optimizer.optimize
+  class PromptOptimizer
+    # Minimum number of examples required for optimization
+    MINIMUM_EXAMPLES = 10
+
+    # Available optimizers with their strategy classes
+    OPTIMIZERS = {
+      bootstrap: {
+        name: "Bootstrap",
+        strategy_class: Leva::Optimizers::Bootstrap,
+        gem: nil,
+        description: "Fast and simple. Automatically selects optimal few-shot examples from your dataset. " \
+                     "Best for quick iteration and when you have limited data (10-50 examples). " \
+                     "Does not modify instructions, only adds demonstrations."
+      },
+      gepa: {
+        name: "GEPA",
+        strategy_class: Leva::Optimizers::GepaOptimizer,
+        gem: "dspy-gepa",
+        description: "State-of-the-art optimizer using reflective prompt evolution. Uses LLM reflection " \
+                     "to identify what works and propose improvements. Outperforms MIPROv2 by 10-14% " \
+                     "while being more sample efficient. Best choice for maximum quality."
+      },
+      miprov2: {
+        name: "MIPROv2",
+        strategy_class: Leva::Optimizers::Miprov2Optimizer,
+        gem: "dspy-miprov2",
+        description: "Uses Bayesian optimization to search for optimal instructions and few-shot examples. " \
+                     "Good for larger datasets (200+ examples). More computationally demanding but thorough. " \
+                     "Can overfit on small datasets."
+      }
+    }.freeze
+
+    # Default optimizer
+    DEFAULT_OPTIMIZER = :bootstrap
+
+    # Optimization modes with their approximate durations
+    MODES = {
+      light: { description: "Fast optimization (~5 min)", trials: 5 },
+      medium: { description: "Balanced optimization (~15 min)", trials: 15 },
+      heavy: { description: "Thorough optimization (~30 min)", trials: 30 }
+    }.freeze
+
+    # Default model if none specified (fast and cheap)
+    DEFAULT_MODEL = "gemini-2.5-flash"
+
+    # Returns available models from RubyLLM.
+    # Results are cached for 5 minutes to avoid repeated expensive calls.
+    #
+    # @return [Array<RubyLLM::Model>] All available chat models
+    def self.available_models
+      Rails.cache.fetch("leva/available_models", expires_in: 5.minutes) do
+        RubyLLM.models.chat_models
+      end
+    end
+
+    # Finds a model by ID.
+    #
+    # @param model_id [String] The model ID to find
+    # @return [RubyLLM::Model, nil] The model or nil if not found
+    def self.find_model(model_id)
+      RubyLLM.models.find(model_id)
+    rescue RubyLLM::ModelNotFoundError
+      nil
+    end
+
+    # @return [Leva::Dataset] The dataset being optimized
+    attr_reader :dataset
+
+    # @return [Symbol] The optimization mode (:light, :medium, :heavy)
+    attr_reader :mode
+
+    # @return [String] The model to use for optimization
+    attr_reader :model
+
+    # @return [Symbol] The optimizer to use (:bootstrap, :gepa, :miprov2)
+    attr_reader :optimizer
+
+    # @param dataset [Leva::Dataset] The dataset to optimize for
+    # @param metric [Proc, nil] Custom evaluation metric (default: exact string match)
+    # @param mode [Symbol] Optimization intensity (:light, :medium, :heavy)
+    # @param model [String, nil] The model to use (default: DEFAULT_MODEL)
+    # @param optimizer [Symbol, String] The optimizer to use (default: :bootstrap)
+    # @param progress_callback [Proc, nil] Callback for progress updates
+    def initialize(dataset:, metric: nil, mode: :light, model: nil, optimizer: nil, progress_callback: nil)
+      @dataset = dataset
+      @metric = metric || default_metric
+      @mode = mode.to_sym
+      @model = model.presence || DEFAULT_MODEL
+      @optimizer = (optimizer.presence || DEFAULT_OPTIMIZER).to_sym
+      @progress_callback = progress_callback
+      @last_progress = nil
+    end
+
+    # Runs the optimization process.
+    #
+    # @return [Hash] Hash containing :system_prompt, :user_prompt, and :metadata
+    # @raise [Leva::InsufficientDataError] If dataset has too few records
+    # @raise [Leva::DspyConfigurationError] If DSPy is not configured
+    def optimize
+      report_progress(step: "validating", progress: 0)
+      validate_dataset!
+      validate_dspy_configuration!
+      validate_optimizer!
+
+      report_progress(step: "splitting_data", progress: 10)
+      splits = DatasetConverter.new(@dataset).split
+
+      report_progress(step: "generating_signature", progress: 20)
+      signature = SignatureGenerator.new(@dataset).generate
+
+      # Delegate to optimizer strategy
+      strategy = build_optimizer_strategy
+      result = strategy.optimize(splits, signature)
+
+      report_progress(step: "complete", progress: 100)
+
+      build_final_result(result, splits, strategy.optimizer_type)
+    end
+
+    # Checks if the dataset is ready for optimization.
+    #
+    # @return [Boolean] True if the dataset can be optimized
+    def can_optimize?
+      @dataset.dataset_records.count >= MINIMUM_EXAMPLES
+    end
+
+    # Returns the number of additional records needed for optimization.
+    #
+    # @return [Integer] Number of records still needed (0 if ready)
+    def records_needed
+      [ MINIMUM_EXAMPLES - @dataset.dataset_records.count, 0 ].max
+    end
+
+    # Checks if a specific optimizer is available.
+    #
+    # @param optimizer_type [Symbol] The optimizer to check
+    # @return [Boolean] True if the optimizer is available
+    def self.optimizer_available?(optimizer_type)
+      optimizer_type = optimizer_type.to_sym
+      return true if optimizer_type == :bootstrap
+
+      case optimizer_type
+      when :gepa
+        !!defined?(DSPy::Teleprompt::GEPA)
+      when :miprov2
+        !!defined?(DSPy::Teleprompt::MIPROv2)
+      else
+        false
+      end
+    end
+
+    private
+
+    # Builds the optimizer strategy instance.
+    #
+    # @return [Leva::Optimizers::Base] The optimizer strategy
+    def build_optimizer_strategy
+      config = OPTIMIZERS[@optimizer]
+      config[:strategy_class].new(
+        model: @model,
+        metric: @metric,
+        mode: @mode,
+        progress_callback: @progress_callback
+      )
+    end
+
+    # Builds the final result hash from optimization.
+    #
+    # @param result [Hash] The optimizer result with :instruction, :few_shot_examples, :score
+    # @param splits [Hash] The data splits
+    # @param optimizer_type [Symbol] The optimizer that was used
+    # @return [Hash] The formatted result
+    def build_final_result(result, splits, optimizer_type)
+      sample_record = @dataset.dataset_records.first&.recordable
+      input_fields = sample_record&.to_llm_context&.keys || []
+
+      formatted_examples = result[:few_shot_examples].map do |ex|
+        { input: ex[:input], output: ex.dig(:expected, :output) }
+      end
+
+      {
+        system_prompt: result[:instruction],
+        user_prompt: build_user_prompt_template(input_fields),
+        metadata: {
+          optimization: {
+            score: result[:score],
+            mode: @mode.to_s,
+            optimizer: optimizer_type.to_s,
+            model: @model,
+            few_shot_examples: formatted_examples,
+            optimized_at: Time.current.iso8601,
+            dataset_size: @dataset.dataset_records.count,
+            train_size: splits[:train].size,
+            val_size: splits[:val].size,
+            test_size: splits[:test].size
+          }
+        }
+      }
+    end
+
+    # Reports progress to the callback if provided.
+    # Throttles updates to only report when progress changes by 5% or more.
+    #
+    # @param step [String] Current step name
+    # @param progress [Integer] Progress percentage (0-100)
+    # @param examples_processed [Integer, nil] Number of examples processed
+    # @param total [Integer, nil] Total examples to process
+    # @return [void]
+    def report_progress(step:, progress:, examples_processed: nil, total: nil)
+      return unless @progress_callback
+
+      # Skip if progress hasn't changed by at least 5%
+      return if @last_progress && (progress - @last_progress).abs < 5
+
+      @last_progress = progress
+      @progress_callback.call(
+        step: step,
+        progress: progress,
+        examples_processed: examples_processed,
+        total: total
+      )
+    end
+
+    # Validates that the dataset has enough records.
+    #
+    # @raise [Leva::InsufficientDataError] If dataset has too few records
+    def validate_dataset!
+      count = @dataset.dataset_records.count
+      return if count >= MINIMUM_EXAMPLES
+
+      raise InsufficientDataError,
+        "Dataset needs at least #{MINIMUM_EXAMPLES} records for optimization, has #{count}"
+    end
+
+    # Validates that DSPy is properly configured.
+    #
+    # @raise [Leva::DspyConfigurationError] If DSPy is not configured
+    def validate_dspy_configuration!
+      unless defined?(DSPy) && defined?(DSPy::Predict)
+        raise DspyConfigurationError, "DSPy is not installed. Add 'dspy' gem to your Gemfile."
+      end
+    end
+
+    # Validates that the selected optimizer is available.
+    #
+    # @raise [Leva::DspyConfigurationError] If optimizer is not available
+    def validate_optimizer!
+      return if @optimizer == :bootstrap
+      return if self.class.optimizer_available?(@optimizer)
+
+      gem_name = OPTIMIZERS.dig(@optimizer, :gem)
+      raise DspyConfigurationError, <<~MSG.strip
+        #{@optimizer.to_s.upcase} optimizer is not available. Install it:
+
+          gem 'dspy'
+          gem '#{gem_name}'
+
+        Or set DSPY_WITH_#{@optimizer.to_s.upcase}=1 before requiring dspy.
+      MSG
+    end
+
+    # Returns the default evaluation metric (case-insensitive exact match).
+    # Handles both Hash examples and DSPy::Example objects.
+    #
+    # @return [Proc] The default metric function
+    def default_metric
+      lambda do |example, prediction|
+        # Handle both Hash and DSPy::Example
+        expected_output = if example.is_a?(Hash)
+                            example.dig(:expected, :output)
+        else
+                            # DSPy::Example has expected_values method to get Hash
+                            example.expected_values[:output]
+        end
+        expected = expected_output.to_s.strip.downcase
+        actual = prediction.to_s.strip.downcase
+        expected == actual ? 1.0 : 0.0
+      end
+    end
+
+    # Builds the user prompt template with Liquid placeholders.
+    #
+    # @param input_fields [Array<Symbol>] The input field names
+    # @return [String] The user prompt template
+    def build_user_prompt_template(input_fields)
+      input_fields.map { |field| "{{ #{field} }}" }.join("\n\n")
+    end
+  end
+end