leva 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0019e85833a0d18992a78748b1726d696c77b0fbc778f2ccb1b8f2916f6ed302'
-  data.tar.gz: e6dd98f6ec9fcb6067d27ee49a482e921ad69154a62fbef867b90d11af19be9f
+  metadata.gz: '03694d16308b610d8c1cc83ec070cf2c0a03273d93b4e220834ff063f8df5b0a'
+  data.tar.gz: 31fa8e5737410dbb9b5729bf43616ef037fbad1c6b8188e60649a5156c8f87c1
 SHA512:
-  metadata.gz: 9ad48cafaf84e3a4899397e4b9eb43c2f1868eded4761afd114e297ca265fd8abd19cf8d29edfe625a93994bff406818d4adf4cb35da5faa54ff17000276eca0
-  data.tar.gz: e506343e51badf77c050a4aa26abdfcf442fa84432d6f5aaa9f3fd4019a0f933a0d5a2b66fab18a6f89827f27803bdadfbbffd7b79ddf9c4f76187e4dc53ade6
+  metadata.gz: f12f9ec8d00a5dcd9a8c003a598d9ec316be4bd8b8b2deb7a99680a14dcd64790b496829e7635e28f5b86dd7a5f484b9043b504bda24f7e3d0fd75b8e4eee271
+  data.tar.gz: 293f53edc39d95ed612b0ce0e0e5097f38e888990c7e8530b54da6afcf2015ae7f150f8f9bd9d2bb1171c5bf18c0c4a34180482594c376ed17341ae42bce9f09

data/README.md

@@ -5,7 +5,7 @@
 
 Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets on production models. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic on production data with security in mind.
 
-![Leva - Workbench- Google Chrome@2x](https://github.com/user-attachments/assets/1631a8f7-0634-4554-8f8b-e643062378a8)
+![✳ Mac Battery Drain- Warp@2x](https://github.com/user-attachments/assets/6c2ea720-a5ab-4ec4-9272-ee50114fa9f6)
 
 ## Installation
 
@@ -181,6 +181,60 @@ experiment.evaluation_results.group_by(&:evaluator_class).each do |evaluator_cla
 end
 ```
 
+## Prompt Optimization (DSPy Integration)
+
+Leva includes optional prompt optimization powered by [DSPy.rb](https://github.com/kieranklaassen/dspy.rb). This feature automatically finds optimal prompts and few-shot examples for your datasets.
+
+**Requirements:**
+- Ruby 3.3.0 or higher
+- DSPy gem and optional optimizer gems
+
+### Installation
+
+Add the DSPy gems to your Gemfile:
+
+```ruby
+gem "dspy"           # Core DSPy functionality (required)
+gem "dspy-gepa"      # GEPA optimizer (optional, recommended)
+gem "dspy-miprov2"   # MIPROv2 optimizer (optional)
+```
+
+### Available Optimizers
+
+| Optimizer | Best For | Description |
+|-----------|----------|-------------|
+| **Bootstrap** | Quick iteration, small datasets | Fast selection of few-shot examples. No gem required. |
+| **GEPA** | Maximum quality | State-of-the-art reflective prompt evolution. 10-14% better than MIPROv2. |
+| **MIPROv2** | Large datasets (200+) | Bayesian optimization for instructions and examples. |
+
+### Usage
+
+```ruby
+# Create an optimizer for your dataset
+optimizer = Leva::PromptOptimizer.new(
+  dataset: dataset,
+  optimizer: :gepa,      # :bootstrap, :gepa, or :miprov2
+  mode: :medium,         # :light, :medium, or :heavy
+  model: "gpt-4o-mini"   # Any model supported by RubyLLM
+)
+
+# Run optimization
+result = optimizer.optimize
+
+# Result contains optimized prompts
+result[:system_prompt]  # Optimized instruction
+result[:user_prompt]    # Template with Liquid variables
+result[:metadata]       # Score, examples, and optimization details
+```
+
+### Optimization Modes
+
+| Mode | Duration | Use Case |
+|------|----------|----------|
+| `:light` | ~5 min | Quick experiments |
+| `:medium` | ~15 min | Balanced quality/speed |
+| `:heavy` | ~30 min | Production prompts |
+
 ## Configuration
 
 Ensure you set up any required API keys or other configurations in your Rails credentials or environment variables.

data/app/assets/stylesheets/leva/application.css

@@ -947,6 +947,15 @@ strong {
   color: var(--info-400);
 }
 
+.badge-optimized {
+  background: rgba(168, 127, 219, 0.15);
+  color: #b794f4;
+  font-size: 0.65rem;
+  padding: 2px 6px;
+  margin-left: 6px;
+  vertical-align: middle;
+}
+
 /* Status Dots */
 .status-dot {
   display: inline-block;
@@ -1330,6 +1339,13 @@ strong {
   padding: var(--space-4);
   overflow-y: auto;
   transition: width var(--transition-base), padding var(--transition-base);
+  /* Hide scrollbar while maintaining scroll */
+  scrollbar-width: none;
+  -ms-overflow-style: none;
+}
+
+.panel-right::-webkit-scrollbar {
+  display: none;
 }
 
 .panel-right.resizing {
@@ -1833,6 +1849,40 @@ dialog::backdrop {
 }
 
 
+/* ============================================
+   SCROLLBAR HIDING (while maintaining scroll)
+   ============================================ */
+
+/* Hide scrollbars globally in workbench panels */
+.panel,
+.panel-right,
+.sidebar-content,
+.output-block,
+.result-block,
+.main-content,
+.prompt-textarea,
+.prompt-sidebar,
+.layout-workbench,
+.content-body,
+.output-value {
+  scrollbar-width: none; /* Firefox */
+  -ms-overflow-style: none; /* IE 10+ */
+}
+
+.panel::-webkit-scrollbar,
+.panel-right::-webkit-scrollbar,
+.sidebar-content::-webkit-scrollbar,
+.output-block::-webkit-scrollbar,
+.result-block::-webkit-scrollbar,
+.main-content::-webkit-scrollbar,
+.prompt-textarea::-webkit-scrollbar,
+.prompt-sidebar::-webkit-scrollbar,
+.layout-workbench::-webkit-scrollbar,
+.content-body::-webkit-scrollbar,
+.output-value::-webkit-scrollbar {
+  display: none; /* Chrome, Safari, Opera */
+}
+
 /* ============================================
    WORKBENCH SPECIFIC
    ============================================ */
@@ -1993,9 +2043,11 @@ dialog::backdrop {
    ============================================ */
 
 .run-controls {
-  padding-bottom: var(--space-4);
-  border-bottom: 1px solid var(--gray-800);
-  margin-bottom: var(--space-4);
+  padding: var(--space-4);
+  background: rgba(26, 25, 24, 0.4);
+  border-radius: var(--radius-lg);
+  border: 1px solid var(--gray-800);
+  margin-bottom: var(--space-3);
 }
 
 .run-selects {
@@ -2024,22 +2076,26 @@ dialog::backdrop {
    ============================================ */
 
 .output-section {
-  padding-bottom: var(--space-4);
-  border-bottom: 1px solid var(--gray-800);
-  margin-bottom: var(--space-4);
+  padding: var(--space-4);
+  background: rgba(26, 25, 24, 0.4);
+  border-radius: var(--radius-lg);
+  margin-bottom: var(--space-3);
+  border: 1px solid var(--gray-800);
 }
 
 .output-header {
   display: flex;
-  align-items: baseline;
+  align-items: center;
   justify-content: space-between;
-  margin-bottom: var(--space-3);
+  margin-bottom: var(--space-4);
+  padding-bottom: var(--space-3);
+  border-bottom: 1px solid var(--gray-800);
 }
 
 .output-title {
   font-size: var(--text-xs);
   font-weight: 600;
-  color: var(--gray-400);
+  color: var(--gray-300);
   text-transform: uppercase;
   letter-spacing: var(--tracking-wide);
 }
@@ -2047,48 +2103,125 @@ dialog::backdrop {
 .output-meta {
   font-size: 10px;
   color: var(--gray-500);
+  display: flex;
+  align-items: center;
+  gap: var(--space-2);
 }
 
 .output-grid {
   display: flex;
   flex-direction: column;
-  gap: var(--space-1);
+  gap: var(--space-3);
 }
 
 .output-block {
   background: var(--gray-800);
-  border-radius: var(--radius-sm);
-  padding: var(--space-2) var(--space-3);
+  border-radius: var(--radius-md);
+  padding: var(--space-3) var(--space-4);
   display: flex;
-  align-items: baseline;
+  align-items: flex-start;
   gap: var(--space-3);
+  transition: background-color 0.15s ease, border-color 0.15s ease;
+}
+
+.output-block:hover {
+  background: var(--gray-750, #2a2928);
 }
 
 .output-block--expected {
-  background: rgba(125, 179, 103, 0.08);
-  border-left: 2px solid var(--success-500);
+  background: rgba(125, 179, 103, 0.06);
+  border-left: 3px solid var(--success-500);
+  padding: var(--space-2) var(--space-3);
+}
+
+.output-block--expected:hover {
+  background: rgba(125, 179, 103, 0.1);
+}
+
+.output-block--expected .output-value {
+  font-weight: 500;
+  color: var(--gray-50);
 }
 
 .output-block--got {
-  border-left: 2px solid var(--gray-700);
+  background: var(--gray-850, #222120);
+  border: 1px solid var(--gray-700);
+  border-left: 3px solid var(--gray-600);
+}
+
+.output-block--parsed {
+  background: rgba(212, 168, 74, 0.06);
+  border-left: 3px solid var(--accent-500);
+  padding: var(--space-2) var(--space-3);
+}
+
+.output-block--parsed:hover {
+  background: rgba(212, 168, 74, 0.1);
+}
+
+.output-block--parsed .output-value {
+  font-weight: 500;
+  color: var(--accent-300);
 }
 
 .output-label {
   font-size: 10px;
   font-weight: 600;
-  color: var(--gray-500);
+  color: var(--gray-400);
   text-transform: uppercase;
-  letter-spacing: 0.03em;
+  letter-spacing: 0.05em;
   flex-shrink: 0;
-  width: 56px;
+  min-width: 64px;
+  padding-top: 2px;
 }
 
 .output-value {
   font-family: var(--font-mono);
   font-size: var(--text-sm);
   color: var(--gray-100);
-  line-height: 1.5;
+  line-height: 1.6;
   word-break: break-word;
+  white-space: pre-wrap;
+  flex: 1;
+  min-width: 0;
+}
+
+/* Long output values get constrained height with scroll */
+.output-block--got .output-value {
+  max-height: 180px;
+  overflow-y: auto;
+  scrollbar-width: none;
+  -ms-overflow-style: none;
+  font-size: 12px;
+  line-height: 1.7;
+  color: var(--gray-200);
+}
+
+.output-block--got .output-value::-webkit-scrollbar {
+  display: none;
+}
+
+/* Fade effect for long content */
+.output-block--got {
+  position: relative;
+}
+
+.output-block--got::after {
+  content: '';
+  position: absolute;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  height: 40px;
+  background: linear-gradient(transparent, var(--gray-850, #222120));
+  pointer-events: none;
+  border-radius: 0 0 var(--radius-md) var(--radius-md);
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.output-block--got:has(.output-value:not(:hover)):after {
+  opacity: 0.8;
 }
 
 .output-empty {
@@ -2103,20 +2236,25 @@ dialog::backdrop {
    ============================================ */
 
 .eval-section {
-  /* No bottom border - last section */
+  padding: var(--space-4);
+  background: rgba(26, 25, 24, 0.4);
+  border-radius: var(--radius-lg);
+  border: 1px solid var(--gray-800);
 }
 
 .eval-header {
   display: flex;
   align-items: center;
   justify-content: space-between;
-  margin-bottom: var(--space-3);
+  margin-bottom: var(--space-4);
+  padding-bottom: var(--space-3);
+  border-bottom: 1px solid var(--gray-800);
 }
 
 .eval-title {
   font-size: var(--text-xs);
   font-weight: 600;
-  color: var(--gray-400);
+  color: var(--gray-300);
   text-transform: uppercase;
   letter-spacing: var(--tracking-wide);
 }
@@ -2130,8 +2268,8 @@ dialog::backdrop {
 .eval-card {
   flex: 1 1 calc(50% - var(--space-1));
   min-width: 0;
-  background: var(--gray-800);
-  border: 1px solid transparent;
+  background: var(--gray-850, #222120);
+  border: 1px solid var(--gray-700);
   border-radius: var(--radius-md);
   padding: var(--space-3);
   cursor: pointer;
@@ -2143,6 +2281,8 @@ dialog::backdrop {
 
 .eval-card:hover {
   background: var(--gray-750, #2a2928);
+  border-color: var(--gray-600);
+  transform: translateY(-1px);
 }
 
 .eval-card-inner {

data/app/controllers/leva/dataset_optimizations_controller.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Leva
+  class DatasetOptimizationsController < ApplicationController
+    before_action :set_dataset
+
+    # GET /datasets/:dataset_id/optimization/new
+    # Shows the prompt optimization form
+    # @return [void]
+    def new
+      @record_count = @dataset.dataset_records.count
+      @prompt_optimizer = PromptOptimizer.new(dataset: @dataset)
+      @can_optimize = @prompt_optimizer.can_optimize?
+      @records_needed = @prompt_optimizer.records_needed
+      @modes = PromptOptimizer::MODES
+      @models = PromptOptimizer.available_models
+      @optimizers = PromptOptimizer::OPTIMIZERS
+    end
+
+    # POST /datasets/:dataset_id/optimization
+    # Starts the prompt optimization job with progress tracking
+    # @return [void]
+    def create
+      opt_params = optimization_params
+
+      @optimization_run = @dataset.optimization_runs.create!(
+        prompt_name: opt_params[:prompt_name],
+        mode: opt_params[:mode],
+        model: opt_params[:model],
+        optimizer: opt_params[:optimizer],
+        status: :pending
+      )
+
+      PromptOptimizationJob.perform_later(optimization_run_id: @optimization_run.id)
+
+      redirect_to optimization_run_path(@optimization_run)
+    end
+
+    private
+
+    # Use callbacks to share common setup or constraints between actions.
+    # @return [void]
+    def set_dataset
+      @dataset = Dataset.find(params[:dataset_id])
+    end
+
+    # Strong parameters for optimization run creation.
+    # @return [Hash]
+    # @raise [ActionController::BadRequest] If model is invalid
+    def optimization_params
+      model = params[:model].presence || PromptOptimizer::DEFAULT_MODEL
+      unless PromptOptimizer.find_model(model)
+        raise ActionController::BadRequest, "Invalid model: #{model}"
+      end
+
+      {
+        prompt_name: params[:prompt_name].presence || "Optimized: #{@dataset.name}",
+        mode: params[:mode].presence || "light",
+        model: model,
+        optimizer: params[:optimizer].presence || PromptOptimizer::DEFAULT_OPTIMIZER.to_s
+      }
+    end
+  end
+end

data/app/controllers/leva/experiments_controller.rb

@@ -2,8 +2,6 @@
 
 module Leva
   class ExperimentsController < ApplicationController
-    include ApplicationHelper
-
     before_action :set_experiment, only: [ :show, :edit, :update ]
     before_action :check_editable, only: [ :edit, :update ]
     before_action :load_runners_and_evaluators, only: [ :new, :edit, :create, :update ]
@@ -11,7 +9,8 @@ module Leva
     # GET /experiments
     # @return [void]
     def index
-      @experiments = Experiment.all
+      @experiments = Experiment.includes(:evaluation_results).all
+      @evaluator_classes = Leva::EvaluationResult.distinct.pluck(:evaluator_class)
     end
 
     # GET /experiments/1
@@ -83,12 +82,21 @@ module Leva
     # Only allow a list of trusted parameters through.
     # @return [ActionController::Parameters]
     def experiment_params
-      params.require(:experiment).permit(:name, :description, :dataset_id, :prompt_id, :runner_class, evaluator_classes: [])
+      permitted = params.require(:experiment).permit(:name, :description, :dataset_id, :prompt_id, :runner_class, evaluator_classes: [], metadata: {})
+      # Ensure metadata is a hash, not ActionController::Parameters
+      if permitted[:metadata].present?
+        metadata_hash = permitted[:metadata].to_h
+        if metadata_hash.to_json.bytesize > 100_000
+          raise ActionController::BadRequest, "Metadata exceeds maximum size of 100KB"
+        end
+        permitted[:metadata] = metadata_hash
+      end
+      permitted
     end
 
     def load_runners_and_evaluators
-      @runners = load_runners
-      @evaluators = load_evaluators
+      @runners = Leva::ClassLoader.runners
+      @evaluators = Leva::ClassLoader.evaluators
     end
 
     def check_editable

data/app/controllers/leva/workbench_controller.rb

@@ -19,14 +19,12 @@ module Leva
       @selected_dataset_record = params[:dataset_record_id] || DatasetRecord.first&.id
 
       # Get merged context if runner and dataset record are available
-      if @selected_runner && @dataset_record
-        runner_class = @selected_runner.constantize rescue nil
-        if runner_class && runner_class < Leva::BaseRun
-          runner = runner_class.new
-          @record_context = @dataset_record.recordable.to_llm_context
-          @runner_context = runner.to_llm_context(@dataset_record.recordable)
-          @merged_context = @record_context.merge(@runner_context)
-        end
+      if @selected_runner && @dataset_record && valid_runner?(@selected_runner)
+        runner_class = @selected_runner.constantize
+        runner = runner_class.new
+        @record_context = @dataset_record.recordable.to_llm_context
+        @runner_context = runner.to_llm_context(@dataset_record.recordable)
+        @merged_context = @record_context.merge(@runner_context)
       end
     end
 
@@ -67,8 +65,8 @@ module Leva
     def run
       return redirect_to workbench_index_path, alert: "Please select a record and a runner" unless @dataset_record && run_params[:runner]
 
+      return redirect_to workbench_index_path, alert: "Invalid runner selected" unless valid_runner?(run_params[:runner])
       runner_class = run_params[:runner].constantize
-      return redirect_to workbench_index_path, alert: "Invalid runner selected" unless runner_class < Leva::BaseRun
 
       runner = runner_class.new
       runner_result = runner.execute_and_store(nil, @dataset_record, @prompt)
@@ -90,8 +88,8 @@ module Leva
     def run_evaluator
       return redirect_to workbench_index_path, alert: "No runner result available" unless @runner_result
 
+      return redirect_to workbench_index_path, alert: "Invalid evaluator selected" unless allowed_evaluator_names.include?(params[:evaluator])
       evaluator_class = params[:evaluator].constantize
-      return redirect_to workbench_index_path, alert: "Invalid evaluator selected" unless evaluator_class < Leva::BaseEval
 
       evaluator = evaluator_class.new
       evaluator.evaluate_and_store(nil, @runner_result)
@@ -120,5 +118,23 @@ module Leva
     def set_runner_result
       @runner_result = @dataset_record.runner_results.last if @dataset_record
     end
+
+    def allowed_runner_names
+      @allowed_runner_names ||= load_runners.map(&:name)
+    end
+
+    def allowed_evaluator_names
+      @allowed_evaluator_names ||= load_evaluators.map(&:name)
+    end
+
+    def valid_runner?(runner_name)
+      return true if allowed_runner_names.include?(runner_name)
+
+      # Also accept any class that inherits from BaseRun (for testing)
+      klass = runner_name.constantize
+      klass < Leva::BaseRun
+    rescue NameError
+      false
+    end
   end
 end

data/app/helpers/leva/application_helper.rb

@@ -4,14 +4,44 @@ module Leva
     #
     # @return [Array<Class>] An array of evaluator classes
     def load_evaluators
-      load_classes_from_directory("app/evals", Leva::BaseEval) || []
+      Leva::ClassLoader.evaluators
     end
 
     # Loads all runner classes that inherit from Leva::BaseRun
     #
     # @return [Array<Class>] An array of runner classes
     def load_runners
-      load_classes_from_directory("app/runners", Leva::BaseRun) || []
+      Leva::ClassLoader.runners
+    end
+
+    # Returns the CSS class for a score value.
+    #
+    # @param score [Float, nil] The score value (0.0 - 1.0)
+    # @return [String] The CSS class for the score
+    def score_class(score)
+      return "" if score.nil?
+
+      case score
+      when 0...0.2 then "score-bad"
+      when 0.2...0.4 then "score-poor"
+      when 0.4...0.6 then "score-fair"
+      when 0.6...0.8 then "score-good"
+      else "score-excellent"
+      end
+    end
+
+    # Returns the display name for a model.
+    #
+    # Uses RubyLLM to find the model and get its display name,
+    # falling back to extracting the name from the model ID.
+    #
+    # @param model_id [String] The model ID
+    # @return [String] The display name for the model
+    def model_display_name(model_id)
+      return "—" if model_id.blank?
+
+      @models_cache ||= Leva::PromptOptimizer.available_models.index_by(&:id)
+      @models_cache[model_id]&.name || model_id.split("/").last
     end
 
     # Loads predefined prompts from markdown files
@@ -25,19 +55,5 @@ module Leva
       end
       prompts
     end
-
-    private
-
-    # Loads classes from a specified directory that inherit from a given base class
-    #
-    # @param directory [String] The directory path to load classes from
-    # @param base_class [Class] The base class that loaded classes should inherit from
-    # @return [Array<Class>] An array of loaded classes
-    def load_classes_from_directory(directory, base_class)
-      classes = Dir[Rails.root.join(directory, "*.rb")].map do |file|
-        File.basename(file, ".rb").camelize.constantize
-      end.select { |klass| klass < base_class }
-      classes.empty? ? [] : classes
-    end
   end
 end

data/app/models/leva/dataset.rb

@@ -12,6 +12,7 @@ module Leva
   class Dataset < ApplicationRecord
     has_many :dataset_records, dependent: :destroy
     has_many :experiments, dependent: :destroy
+    has_many :optimization_runs, dependent: :destroy
 
     validates :name, presence: true
 

data/app/models/leva/experiment.rb

@@ -39,5 +39,6 @@ module Leva
     enum :status, { pending: 0, running: 1, completed: 2, failed: 3 }, default: :pending
 
     serialize :evaluator_classes, coder: JSON, type: Array
+    serialize :metadata, coder: JSON
   end
 end