leva 0.2.1 → 0.3.1

data/app/services/leva/signature_generator.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Leva
+  # Generates DSPy signatures from Leva dataset records.
+  #
+  # This service analyzes the structure of dataset records and generates
+  # a dynamic DSPy::Signature class that matches the input/output schema.
+  #
+  # @example Generate a signature from a dataset
+  #   generator = Leva::SignatureGenerator.new(dataset)
+  #   signature_class = generator.generate
+  #   predictor = DSPy::Predict.new(signature_class)
+  class SignatureGenerator
+    # @param dataset [Leva::Dataset] The dataset to analyze
+    # @param description [String, nil] Optional description for the signature
+    def initialize(dataset, description: nil)
+      @dataset = dataset
+      @description = description
+      @sample_record = dataset.dataset_records.first&.recordable
+    end
+
+    # Generates a DSPy::Signature class based on the dataset structure.
+    #
+    # @return [Class, nil] A dynamically generated DSPy::Signature subclass, or nil if no sample
+    def generate
+      return nil unless @sample_record
+
+      input_fields = extract_input_fields
+      output_type = infer_output_type(@sample_record.ground_truth)
+      description = @description || generate_description
+
+      build_signature_class(input_fields, output_type, description)
+    end
+
+    # Returns the input field names that will be used in the signature.
+    #
+    # @return [Array<Symbol>] Array of input field names
+    def input_field_names
+      return [] unless @sample_record
+
+      extract_input_fields.keys
+    end
+
+    private
+
+    # Extracts input fields from the sample record's LLM context.
+    #
+    # @return [Hash<Symbol, Class>] Map of field names to their inferred types
+    def extract_input_fields
+      context = @sample_record.to_llm_context
+      context.transform_values { |value| infer_type(value) }
+    end
+
+    # Infers the Ruby type for a given value.
+    #
+    # @param value [Object] The value to analyze
+    # @return [Class] The inferred type (String, Integer, Float, Array, or Hash)
+    def infer_type(value)
+      case value
+      when String then String
+      when Integer then Integer
+      when Float then Float
+      when Array then Array
+      when Hash then Hash
+      else String
+      end
+    end
+
+    # Infers the output type from ground truth.
+    #
+    # @param ground_truth [Object] The ground truth value to analyze
+    # @return [Symbol] The output type (:string, :array, or :hash)
+    def infer_output_type(ground_truth)
+      case ground_truth
+      when String then :string
+      when Array then :array
+      when Hash then :hash
+      else :string
+      end
+    end
+
+    # Generates a description for the signature based on the dataset.
+    #
+    # @return [String] A descriptive string for the signature
+    def generate_description
+      # Analyze ground truth values to determine task type
+      ground_truths = @dataset.dataset_records.limit(20).map { |r| r.recordable.ground_truth }.compact
+      unique_outputs = ground_truths.uniq
+
+      if unique_outputs.size <= 10
+        # Classification task - be explicit about output format
+        "Classify the input. Respond with ONLY one of these exact values, nothing else: #{unique_outputs.join(', ')}"
+      else
+        # Generation task
+        "Generate output for the given input from dataset: #{@dataset.name}"
+      end
+    end
+
+    # Builds the DSPy::Signature class dynamically.
+    #
+    # @param input_fields [Hash<Symbol, Class>] Input field definitions
+    # @param output_type [Symbol] The output type
+    # @param description [String] Description for the signature
+    # @return [Class] The generated DSPy::Signature subclass
+    # @raise [Leva::DspyConfigurationError] If DSPy is not available
+    def build_signature_class(input_fields, output_type, description)
+      unless defined?(DSPy::Signature)
+        raise DspyConfigurationError, "DSPy is required for signature generation"
+      end
+
+      captured_input_fields = input_fields
+      captured_description = description
+
+      Class.new(DSPy::Signature) do
+        description captured_description
+
+        input do
+          captured_input_fields.each do |name, _type|
+            const name, String
+          end
+        end
+
+        output do
+          const :output, String
+        end
+      end
+    end
+  end
+end

data/app/views/leva/datasets/show.html.erb

@@ -84,6 +84,9 @@
     <% end %>
   </section>
 
+  <%# Optimized Prompts Section - TODO: Enable when DSPy routes are added %>
+  <%# This feature is available in the PromptOptimizer service but UI routes are pending %>
+
   <%# Experiments Section %>
   <section>
     <div class="section-header">

data/app/views/leva/experiments/_experiment.html.erb

@@ -8,6 +8,9 @@
     else 'status-dot-pending'
   end
   run_count = experiment.runner_results.count
+
+  # Group evaluation results by evaluator_class to avoid N+1 queries
+  grouped_results = experiment.evaluation_results.group_by(&:evaluator_class)
 %>
 <tr class="experiment-row" onclick="window.location='<%= experiment_path(experiment) %>'">
   <td>
@@ -21,6 +24,9 @@
   <td>
     <span class="cell-dataset"><%= experiment.dataset&.name || '—' %></span>
   </td>
+  <td>
+    <span class="cell-model font-mono text-sm"><%= experiment.metadata&.dig("model") || '—' %></span>
+  </td>
   <td class="text-right text-nowrap">
     <span class="cell-timestamp"><%= time_ago_in_words(experiment.created_at) %></span>
   </td>
@@ -33,22 +39,15 @@
   <td class="text-right">
     <span class="cell-count"><%= run_count %></span>
   </td>
-  <% Leva::EvaluationResult.distinct.pluck(:evaluator_class).each do |evaluator_class| %>
+  <% @evaluator_classes.each do |evaluator_class| %>
     <td class="text-right">
-      <% results = experiment.evaluation_results.where(evaluator_class: evaluator_class) %>
+      <% results = grouped_results[evaluator_class] || [] %>
       <% if results.any? %>
         <%
           avg_score = (results.sum(&:score) / results.size.to_f)
           score_pct = (avg_score * 100).round
-          score_class = case avg_score
-            when 0...0.2 then 'score-bad'
-            when 0.2...0.4 then 'score-poor'
-            when 0.4...0.6 then 'score-fair'
-            when 0.6...0.8 then 'score-good'
-            else 'score-excellent'
-          end
         %>
-        <span class="score-pill <%= score_class %>"><%= score_pct %>%</span>
+        <span class="score-pill <%= score_class(avg_score) %>"><%= score_pct %>%</span>
       <% else %>
         <span class="score-empty">—</span>
       <% end %>

data/app/views/leva/experiments/_form.html.erb

@@ -50,6 +50,16 @@
                       class: "form-select" %>
       <p class="form-hint">The runner executes your model logic for each dataset record.</p>
     </div>
+
+    <div class="form-group" id="model-selection-group">
+      <label for="experiment_metadata_model" class="form-label">Model (for LLM runners)</label>
+      <select name="experiment[metadata][model]" id="experiment_metadata_model" class="form-select">
+        <% Leva::PromptOptimizer.available_models.each do |m| %>
+          <option value="<%= m.id %>" <%= 'selected' if @experiment.metadata&.dig("model") == m.id || (@experiment.metadata.blank? && m.id == "gemini-2.5-flash") %>><%= m.name %></option>
+        <% end %>
+      </select>
+      <p class="form-hint">The AI model to use when running LLM-based runners like SentimentLlmRun.</p>
+    </div>
   </div>
 
   <hr class="form-divider">

data/app/views/leva/experiments/index.html.erb

@@ -21,10 +21,11 @@
             <tr>
               <th>Experiment</th>
               <th style="width: 140px;">Dataset</th>
+              <th style="width: 140px;">Model</th>
               <th class="text-right" style="width: 90px;">Created</th>
               <th class="text-center" style="width: 90px;">Status</th>
               <th class="text-right" style="width: 60px;">Runs</th>
-              <% Leva::EvaluationResult.distinct.pluck(:evaluator_class).each do |evaluator_class| %>
+              <% @evaluator_classes.each do |evaluator_class| %>
                 <%
                   # Clean up evaluator name: "SentimentAccuracyEval" -> "Accuracy"
                   # Remove common prefixes/suffixes and module names

data/app/views/leva/experiments/show.html.erb

@@ -55,12 +55,27 @@
       </div>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Prompt</span>
-        <span class="exp-meta-value"><%= @experiment.prompt ? @experiment.prompt.name : '—' %></span>
+        <span class="exp-meta-value">
+          <% if @experiment.prompt %>
+            <%= @experiment.prompt.name %>
+            <% if @experiment.prompt.optimized? %>
+              <span class="badge badge-optimized" title="Generated by <%= @experiment.prompt.optimizer_name&.titleize || 'optimizer' %>">Optimized</span>
+            <% end %>
+          <% else %>
+            —
+          <% end %>
+        </span>
       </div>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Runner</span>
         <span class="exp-meta-value font-mono text-sm"><%= @experiment.runner_class&.demodulize || '—' %></span>
       </div>
+      <% if @experiment.metadata&.dig("model").present? %>
+        <div class="exp-meta-item">
+          <span class="exp-meta-label">Model</span>
+          <span class="exp-meta-value font-mono text-sm"><%= @experiment.metadata["model"] %></span>
+        </div>
+      <% end %>
       <div class="exp-meta-item">
         <span class="exp-meta-label">Created</span>
         <span class="exp-meta-value"><%= time_ago_in_words(@experiment.created_at) %> ago</span>
@@ -79,13 +94,6 @@
           <%
             avg_score = (results.sum(&:score) / results.size.to_f).round(2)
             score_pct = (avg_score * 100).round
-            score_class = case avg_score
-              when 0...0.2 then 'score-bad'
-              when 0.2...0.4 then 'score-poor'
-              when 0.4...0.6 then 'score-fair'
-              when 0.6...0.8 then 'score-good'
-              else 'score-excellent'
-            end
             short_name = evaluator_class.demodulize
               .gsub(/Evaluator$/, '')
               .gsub(/Eval$/, '')
@@ -93,10 +101,10 @@
             short_name = short_name.presence || evaluator_class.demodulize.gsub(/Eval(uator)?$/, '')
           %>
           <div class="eval-summary-card" title="<%= results.size %> evaluations">
-            <span class="eval-summary-score <%= score_class %>"><%= score_pct %><span class="eval-summary-pct">%</span></span>
+            <span class="eval-summary-score <%= score_class(avg_score) %>"><%= score_pct %><span class="eval-summary-pct">%</span></span>
             <span class="eval-summary-name"><%= short_name %></span>
             <div class="eval-summary-bar">
-              <div class="eval-summary-bar-fill <%= score_class %>" style="width: <%= score_pct %>%"></div>
+              <div class="eval-summary-bar-fill <%= score_class(avg_score) %>" style="width: <%= score_pct %>%"></div>
             </div>
             <span class="eval-summary-count"><%= results.size %> runs</span>
           </div>
@@ -139,7 +147,7 @@
                     <span class="row-title"><%= runner_result.dataset_record.display_name %></span>
                   </td>
                   <td>
-                    <span class="prediction-badge"><%= truncate(runner_result.prediction.to_s.strip, length: 25) %></span>
+                    <span class="prediction-badge"><%= truncate(runner_result.parsed_predictions.first.to_s.presence || runner_result.prediction.to_s.strip, length: 25) %></span>
                   </td>
                   <td class="text-muted"><%= truncate(runner_result.ground_truth.to_s.strip.presence || '—', length: 25) %></td>
                   <% @experiment.evaluation_results.group_by(&:evaluator_class).keys.each do |evaluator_class| %>
@@ -147,16 +155,7 @@
                       <% eval_result = runner_result.evaluation_results.find_by(evaluator_class: evaluator_class) %>
                       <% if eval_result %>
                         <% score = eval_result.score %>
-                        <%
-                          score_class = case score
-                            when 0...0.2 then 'score-bad'
-                            when 0.2...0.4 then 'score-poor'
-                            when 0.4...0.6 then 'score-fair'
-                            when 0.6...0.8 then 'score-good'
-                            else 'score-excellent'
-                          end
-                        %>
-                        <span class="score-inline <%= score_class %>"><%= sprintf('%.2f', score) %></span>
+                        <span class="score-inline <%= score_class(score) %>"><%= sprintf('%.2f', score) %></span>
                       <% else %>
                         <span class="text-subtle">—</span>
                       <% end %>