leva 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2abd3b8bab0e39683850e7e95df8815268332022c2adf6846cd27442e880cb88
-  data.tar.gz: 73c430d4a5a2c6a98dba70a36be988dd40541debbf10b274d62db8e232e931f7
+  metadata.gz: 9d8e48a71a34a53451bc1756a6cdc7694ee6ea9747861fdb977b0859979bd101
+  data.tar.gz: 83c44769f588b3daeeaa6bd750f9e9b643639871de57dd381cd84a76cd6177d6
 SHA512:
-  metadata.gz: ee31e67dae95204cb6c6f2fc96ad41dae624ea48f6760b65571f8f1447fd24ca055a4dba33244067f0138db79a620ec82e8bbb3d3fd4363f9b6e98079b45b6a7
-  data.tar.gz: fadb2838c4d31e498f40ed8e5d20859c57e4e2df433fae1251d3215ddcbc3a2c4843b5a7bd0b73bb4ff4e2b3abb55842b83d7a018ad7feaa40a4da89e6313adc
+  metadata.gz: 74ad608263e8fe369693537d76247757bd39417bbf1e034ffc2402684a9800ed25803d1ff3506d05776c9c523ad7fd84254e78290fc99f21f7180f67e1e12667
+  data.tar.gz: 3a2e4b9701d6cb63c05c2f672b44c8bfd16f0de5a6a98c63e4b6a13de6685efe001cf320304be1fa1aa6c6b56cec658ec0fb95e731fbf078055d5deaffb10959

data/README.md

@@ -1,7 +1,12 @@
 # Leva - Flexible Evaluation Framework for Language Models
 
+[![Gem Version](https://badge.fury.io/rb/leva.svg)](https://badge.fury.io/rb/leva)
+
 Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets on production models. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic on production data with security in mind.
 
+![Leva - Workbench- Google Chrome](https://github.com/user-attachments/assets/ee487941-e11b-4c2a-983b-771ef27dd73c)
+![Leva - rty- Google Chrome](https://github.com/user-attachments/assets/f9986a12-731b-4747-9f86-5ac6fffd5cbc)
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/app/assets/stylesheets/leva/application.css

@@ -13,3 +13,16 @@
  *= require_tree .
  *= require_self
  */
+
+/* Global styles for text overflow handling */
+pre {
+  word-wrap: break-word;
+  word-break: break-word;
+  max-width: 100%;
+}
+
+textarea {
+  word-wrap: break-word;
+  word-break: break-word;
+  max-width: 100%;
+}

data/app/models/leva/dataset_record.rb

@@ -3,6 +3,7 @@
 # Table name: leva_dataset_records
 #
 #  id              :integer          not null, primary key
+#  actual_result   :text
 #  recordable_type :string           not null
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
@@ -61,4 +62,4 @@ module Leva
       end
     end
   end
-end
+end

data/app/models/leva/runner_result.rb

@@ -5,6 +5,7 @@
 #  id                :integer          not null, primary key
 #  prediction        :text
 #  prompt_version    :integer
+#  runner_class      :string
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #  dataset_record_id :integer          not null
@@ -32,23 +33,24 @@ module Leva
 
     validates :prediction, presence: true
     validates :prompt, presence: true
+    validates :runner_class, presence: true
 
     delegate :ground_truth, to: :dataset_record
 
     # @return [Array<String>] The parsed draft responses
     def parsed_predictions
-      @parsed_predictions ||=
-        if extract_regex_pattern
-          prediction.scan(extract_regex_pattern).map { |match| match.first&.strip }.compact
-        else
-          [prediction]
-        end
+      @parsed_predictions ||= runner&.parsed_predictions(self) || []
+    end
+
+    # @return [String] The ground truth for this runner result
+    def ground_truth
+      @ground_truth ||= runner&.ground_truth(self)
     end
 
     private
 
-    def extract_regex_pattern
-      dataset_record.recordable.extract_regex_pattern
+    def runner
+      @runner ||= runner_class&.constantize&.new
     end
   end
-end
+end

data/app/views/leva/workbench/_prompt_content.html.erb

@@ -12,7 +12,7 @@
         </button>
       </div>
       <textarea
-        class="w-full bg-gray-800 text-white p-3 rounded-lg focus:ring-2 focus:ring-indigo-500 focus:outline-none min-h-[100px] overflow-hidden resize-none"
+        class="w-full bg-gray-800 text-white p-3 rounded-lg focus:ring-2 focus:ring-indigo-500 focus:outline-none min-h-[100px] overflow-y-auto resize-none break-words"
         name="prompt[system_prompt]"
         data-prompt-autosave-target="input"
         id="systemPrompt"
@@ -31,7 +31,7 @@
         </button>
       </div>
       <textarea
-        class="w-full bg-gray-800 text-white p-3 rounded-lg focus:ring-2 focus:ring-indigo-500 focus:outline-none min-h-[200px] overflow-hidden resize-none"
+        class="w-full bg-gray-800 text-white p-3 rounded-lg focus:ring-2 focus:ring-indigo-500 focus:outline-none min-h-[200px] overflow-y-auto resize-none break-words"
         name="prompt[user_prompt]"
         data-prompt-autosave-target="input"
         id="userPrompt"
@@ -59,7 +59,7 @@
                   Copy
                 </button>
               </summary>
-              <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap" id="liquidTag<%= key %>"><%= value.to_s %></pre>
+              <pre class="text-xs text-gray-300 mt-1 whitespace-pre-wrap break-words overflow-x-auto max-w-full" id="liquidTag<%= key %>"><%= value.to_s %></pre>
             </details>
           <% end %>
         </div>
@@ -77,7 +77,7 @@
             Copy
           </button>
         </div>
-        <pre class="w-full bg-gray-800 text-white p-3 rounded-lg text-sm whitespace-pre-wrap" id="fullPrompt"><%= Liquid::Template.parse(@selected_prompt.user_prompt).render(@dataset_record.recordable.to_llm_context.stringify_keys) %></pre>
+        <pre class="w-full bg-gray-800 text-white p-3 rounded-lg text-sm whitespace-pre-wrap overflow-x-auto break-words max-w-full" id="fullPrompt"><%= Liquid::Template.parse(@selected_prompt.user_prompt).render(@dataset_record.recordable.to_llm_context.stringify_keys) %></pre>
       </div>
     <% end %>
     <div class="text-sm text-center" data-prompt-autosave-target="status"></div>
@@ -102,7 +102,11 @@
           const textareas = textarea ? [textarea] : this.inputTargets
           textareas.forEach(ta => {
             ta.style.height = 'auto'
-            ta.style.height = ta.scrollHeight + 'px'
+            ta.style.height = (ta.scrollHeight + 5) + 'px'
+            
+            // Ensure horizontal text wrapping
+            ta.style.wordBreak = 'break-word'
+            ta.style.wordWrap = 'break-word'
           })
         }
 

data/app/views/leva/workbench/_results_section.html.erb

@@ -1,4 +1,4 @@
-<div class="w-1/2 bg-gray-900 border-l border-gray-800 p-5 overflow-y-auto" data-controller="button-loader">
+<div class="w-1/2 bg-gray-900 border-l border-gray-800 p-5 overflow-y-auto overflow-x-hidden" data-controller="button-loader">
   <!-- Runner Dropdown -->
   <div class="mb-5">
     <h3 class="text-sm font-semibold mb-2 text-indigo-300">Select Runner</h3>
@@ -42,17 +42,17 @@
     <% if @dataset_record && (runner_result = @dataset_record.runner_results.last) %>
       <div class="mb-3">
         <h4 class="text-xs font-semibold text-indigo-200 mb-1">Ground Truth:</h4>
-        <pre class="text-sm text-gray-300 whitespace-pre-wrap bg-gray-700 p-2 rounded"><%= @dataset_record.ground_truth %></pre>
+        <pre class="text-sm text-gray-300 whitespace-pre-wrap break-words overflow-x-auto max-w-full bg-gray-700 p-2 rounded"><%= runner_result.ground_truth %></pre>
       </div>
       <div>
         <h4 class="text-xs font-semibold text-indigo-200 mb-1">Raw Prediction:</h4>
-        <pre class="text-sm text-gray-300 whitespace-pre-wrap bg-gray-700 p-2 rounded"><%= runner_result.prediction %></pre>
+        <pre class="text-sm text-gray-300 whitespace-pre-wrap break-words overflow-x-auto max-w-full bg-gray-700 p-2 rounded"><%= runner_result.prediction %></pre>
       </div>
       <% if runner_result.dataset_record.recordable.extract_regex_pattern %>
         <div>
           <h4 class="text-xs font-semibold text-indigo-200 my-2 gap-2">Parsed Predictions:  <%= runner_result.dataset_record.recordable.extract_regex_pattern.to_s %></h4>
           <% runner_result.parsed_predictions.each do |prediction| %>
-            <pre class="text-sm text-gray-300 whitespace-pre-wrap bg-gray-700 p-2 rounded mb-2"><%= prediction %></pre>
+            <pre class="text-sm text-gray-300 whitespace-pre-wrap break-words overflow-x-auto max-w-full bg-gray-700 p-2 rounded mb-2"><%= prediction %></pre>
           <% end %>
         </div>
       <% end %>

data/db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb

@@ -0,0 +1,5 @@
+class AddRunnerClassToLevaRunnerResults < ActiveRecord::Migration[7.2]
+  def change
+    add_column :leva_runner_results, :runner_class, :string
+  end
+end

data/lib/generators/leva/templates/eval.rb.erb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 class <%= class_name %>Eval < Leva::BaseEval
-  # @param prediction [String] The prediction to evaluate
+  # @param runner_result [Leva::RunnerResult] The runner result to evaluate
   # @param recordable [YourRecordClass] The recordable object to evaluate
   # @return [Float] The score of the evaluation
-  def evaluate(prediction, recordable)
+  def evaluate(runner_result, recordable)
     # Implement your evaluation logic here
     # You can access the ground truth using recordable.ground_truth
 
     # Example implementation:
-    prediction == recordable.ground_truth ? 1.0 : 0.0
+    runner_result.parsed_predictions.first == recordable.ground_truth ? 1.0 : 0.0
   end
 end

data/lib/generators/leva/templates/runner.rb.erb

@@ -8,4 +8,29 @@ class <%= class_name %>Run < Leva::BaseRun
     # This could involve calling an API, running a local model, etc.
     # Return the result of the run to be used to evaluate the model
   end
+
+  # Uncomment and modify this method to customize parsed predictions
+  # @param runner_result [Leva::RunnerResult] The runner result to parse
+  # @return [Array<String>] The parsed predictions
+  # def parsed_predictions(runner_result)
+  #   # Example: Extract predictions from XML-like tags
+  #   runner_result.prediction.scan(/<prediction>(.*?)<\/prediction>/).flatten
+  # end
+
+  # Uncomment and modify this method to customize ground truth extraction
+  # @param runner_result [Leva::RunnerResult] The runner result to get ground truth from
+  # @return [String] The ground truth for the runner result
+  # def ground_truth(runner_result)
+  #   # Example: Extract ground truth from a specific field
+  #   runner_result.dataset_record.recordable.custom_ground_truth_field
+  # end
+
+  # Uncomment and modify this method to customize regex extraction
+  # @param runner_result [Leva::RunnerResult] The runner result to extract regex from
+  # @return [Regexp, nil] The regex pattern to use for parsing predictions
+  # def extract_regex_pattern(runner_result)
+  #   # Your custom regex extraction logic here
+  #   # For example:
+  #   # /\<result\>(.*?)\<\/result\>/
+  # end
 end

data/lib/leva/version.rb

@@ -1,3 +1,3 @@
 module Leva
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

data/lib/leva.rb

@@ -72,8 +72,31 @@ module Leva
         dataset_record: dataset_record,
         prompt: prompt,
         prediction: result,
+        runner_class: self.class.name
       )
     end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to parse
+    # @return [Array<String>] The parsed predictions
+    def parsed_predictions(runner_result)
+      if extract_regex_pattern(runner_result)
+        runner_result.prediction.scan(extract_regex_pattern(runner_result)).map { |match| match.first&.strip }.compact
+      else
+        [runner_result.prediction]
+      end
+    end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to extract regex from
+    # @return [Regexp, nil] The regex pattern to use for parsing predictions
+    def extract_regex_pattern(runner_result)
+      runner_result.dataset_record.recordable.extract_regex_pattern if runner_result.dataset_record.recordable.respond_to?(:extract_regex_pattern)
+    end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to get ground truth from
+    # @return [String] The ground truth for the runner result
+    def ground_truth(runner_result)
+      runner_result.dataset_record.ground_truth
+    end
   end
 
   # Base class for all evaluation implementations in Leva.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: leva
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Kieran Klaassen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2025-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -111,6 +111,7 @@ files:
 - db/migrate/20240821183153_add_runner_and_evaluator_to_leva_experiments.rb
 - db/migrate/20240821191713_add_actual_result_to_leva_dataset_records.rb
 - db/migrate/20240822143201_remove_actual_result_from_leva_runner_results.rb
+- db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb
 - lib/generators/leva/eval_generator.rb
 - lib/generators/leva/runner_generator.rb
 - lib/generators/leva/templates/eval.rb.erb