leva 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2abd3b8bab0e39683850e7e95df8815268332022c2adf6846cd27442e880cb88
-  data.tar.gz: 73c430d4a5a2c6a98dba70a36be988dd40541debbf10b274d62db8e232e931f7
+  metadata.gz: 0b12a341adbc24fa5c29fe3bf9961345fe02f66fb31089908a61ccf9d4061dff
+  data.tar.gz: 3e74533f1417a0f88aeab8f7942af4df7a6ad0eca6af89c1e29a1cd178cb3121
 SHA512:
-  metadata.gz: ee31e67dae95204cb6c6f2fc96ad41dae624ea48f6760b65571f8f1447fd24ca055a4dba33244067f0138db79a620ec82e8bbb3d3fd4363f9b6e98079b45b6a7
-  data.tar.gz: fadb2838c4d31e498f40ed8e5d20859c57e4e2df433fae1251d3215ddcbc3a2c4843b5a7bd0b73bb4ff4e2b3abb55842b83d7a018ad7feaa40a4da89e6313adc
+  metadata.gz: ad517366dd537adda1d937252e494215d36164f8461dbd818794d746bfb84838e7fa6889e29fa1cd873fdca7992f545832718acb0346ee2cd2dee80bb55ee9df
+  data.tar.gz: 5bf3f499fb42c99c898df934e7c188c85f2d10cb0ad77552b5d7a349552b9f614bc742be52648570ca13da9858e7144ce2dae39196d9a083b8aaac1f6c266b59

data/README.md

@@ -2,6 +2,9 @@
 
 Leva is a Ruby on Rails framework for evaluating Language Models (LLMs) using ActiveRecord datasets on production models. It provides a flexible structure for creating experiments, managing datasets, and implementing various evaluation logic on production data with security in mind.
 
+![Leva - Workbench- Google Chrome](https://github.com/user-attachments/assets/ee487941-e11b-4c2a-983b-771ef27dd73c)
+![Leva - rty- Google Chrome](https://github.com/user-attachments/assets/f9986a12-731b-4747-9f86-5ac6fffd5cbc)
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/app/models/leva/dataset_record.rb

@@ -3,6 +3,7 @@
 # Table name: leva_dataset_records
 #
 #  id              :integer          not null, primary key
+#  actual_result   :text
 #  recordable_type :string           not null
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
@@ -61,4 +62,4 @@ module Leva
       end
     end
   end
-end
+end

data/app/models/leva/runner_result.rb

@@ -5,6 +5,7 @@
 #  id                :integer          not null, primary key
 #  prediction        :text
 #  prompt_version    :integer
+#  runner_class      :string
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #  dataset_record_id :integer          not null
@@ -32,23 +33,24 @@ module Leva
 
     validates :prediction, presence: true
     validates :prompt, presence: true
+    validates :runner_class, presence: true
 
     delegate :ground_truth, to: :dataset_record
 
     # @return [Array<String>] The parsed draft responses
     def parsed_predictions
-      @parsed_predictions ||=
-        if extract_regex_pattern
-          prediction.scan(extract_regex_pattern).map { |match| match.first&.strip }.compact
-        else
-          [prediction]
-        end
+      @parsed_predictions ||= runner&.parsed_predictions(self) || []
+    end
+
+    # @return [String] The ground truth for this runner result
+    def ground_truth
+      @ground_truth ||= runner&.ground_truth(self)
     end
 
     private
 
-    def extract_regex_pattern
-      dataset_record.recordable.extract_regex_pattern
+    def runner
+      @runner ||= runner_class&.constantize&.new
     end
   end
-end
+end

data/app/views/leva/workbench/_results_section.html.erb

@@ -42,7 +42,7 @@
     <% if @dataset_record && (runner_result = @dataset_record.runner_results.last) %>
       <div class="mb-3">
         <h4 class="text-xs font-semibold text-indigo-200 mb-1">Ground Truth:</h4>
-        <pre class="text-sm text-gray-300 whitespace-pre-wrap bg-gray-700 p-2 rounded"><%= @dataset_record.ground_truth %></pre>
+        <pre class="text-sm text-gray-300 whitespace-pre-wrap bg-gray-700 p-2 rounded"><%= runner_result.ground_truth %></pre>
       </div>
       <div>
         <h4 class="text-xs font-semibold text-indigo-200 mb-1">Raw Prediction:</h4>

data/db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb

@@ -0,0 +1,5 @@
+class AddRunnerClassToLevaRunnerResults < ActiveRecord::Migration[7.2]
+  def change
+    add_column :leva_runner_results, :runner_class, :string
+  end
+end

data/lib/generators/leva/templates/eval.rb.erb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 class <%= class_name %>Eval < Leva::BaseEval
-  # @param prediction [String] The prediction to evaluate
+  # @param runner_result [Leva::RunnerResult] The runner result to evaluate
   # @param recordable [YourRecordClass] The recordable object to evaluate
   # @return [Float] The score of the evaluation
-  def evaluate(prediction, recordable)
+  def evaluate(runner_result, recordable)
     # Implement your evaluation logic here
     # You can access the ground truth using recordable.ground_truth
 
     # Example implementation:
-    prediction == recordable.ground_truth ? 1.0 : 0.0
+    runner_result.parsed_predictions.first == recordable.ground_truth ? 1.0 : 0.0
   end
 end

data/lib/generators/leva/templates/runner.rb.erb

@@ -8,4 +8,29 @@ class <%= class_name %>Run < Leva::BaseRun
     # This could involve calling an API, running a local model, etc.
     # Return the result of the run to be used to evaluate the model
   end
+
+  # Uncomment and modify this method to customize parsed predictions
+  # @param runner_result [Leva::RunnerResult] The runner result to parse
+  # @return [Array<String>] The parsed predictions
+  # def parsed_predictions(runner_result)
+  #   # Example: Extract predictions from XML-like tags
+  #   runner_result.prediction.scan(/<prediction>(.*?)<\/prediction>/).flatten
+  # end
+
+  # Uncomment and modify this method to customize ground truth extraction
+  # @param runner_result [Leva::RunnerResult] The runner result to get ground truth from
+  # @return [String] The ground truth for the runner result
+  # def ground_truth(runner_result)
+  #   # Example: Extract ground truth from a specific field
+  #   runner_result.dataset_record.recordable.custom_ground_truth_field
+  # end
+
+  # Uncomment and modify this method to customize regex extraction
+  # @param runner_result [Leva::RunnerResult] The runner result to extract regex from
+  # @return [Regexp, nil] The regex pattern to use for parsing predictions
+  # def extract_regex_pattern(runner_result)
+  #   # Your custom regex extraction logic here
+  #   # For example:
+  #   # /\<result\>(.*?)\<\/result\>/
+  # end
 end

data/lib/leva/version.rb

@@ -1,3 +1,3 @@
 module Leva
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/leva.rb

@@ -72,8 +72,31 @@ module Leva
         dataset_record: dataset_record,
         prompt: prompt,
         prediction: result,
+        runner_class: self.class.name
       )
     end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to parse
+    # @return [Array<String>] The parsed predictions
+    def parsed_predictions(runner_result)
+      if extract_regex_pattern(runner_result)
+        runner_result.prediction.scan(extract_regex_pattern(runner_result)).map { |match| match.first&.strip }.compact
+      else
+        [runner_result.prediction]
+      end
+    end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to extract regex from
+    # @return [Regexp, nil] The regex pattern to use for parsing predictions
+    def extract_regex_pattern(runner_result)
+      runner_result.dataset_record.recordable.extract_regex_pattern if runner_result.dataset_record.recordable.respond_to?(:extract_regex_pattern)
+    end
+
+    # @param runner_result [Leva::RunnerResult] The runner result to get ground truth from
+    # @return [String] The ground truth for the runner result
+    def ground_truth(runner_result)
+      runner_result.dataset_record.ground_truth
+    end
   end
 
   # Base class for all evaluation implementations in Leva.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: leva
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Kieran Klaassen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2024-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -111,6 +111,7 @@ files:
 - db/migrate/20240821183153_add_runner_and_evaluator_to_leva_experiments.rb
 - db/migrate/20240821191713_add_actual_result_to_leva_dataset_records.rb
 - db/migrate/20240822143201_remove_actual_result_from_leva_runner_results.rb
+- db/migrate/20240912183556_add_runner_class_to_leva_runner_results.rb
 - lib/generators/leva/eval_generator.rb
 - lib/generators/leva/runner_generator.rb
 - lib/generators/leva/templates/eval.rb.erb