lex-eval 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dd068d711cd3cc0c70d64f8c066e1bb03e929bc034073600a8e3946c7c65a77
-  data.tar.gz: 6103505a44655acc55a78ac3677b2d8fef300e395d33acb47e5a545cd0f7e8e3
+  metadata.gz: 7986f7010e32abbdf158a8525f35806aee77c3b5e160e0933f436c92a1c7c7ec
+  data.tar.gz: c27e8719b565902494b89ec579cdcd3cf365dd6a223a5dcb7f4089601b35552f
 SHA512:
-  metadata.gz: 4b0ef19e8406c5eaf2914b22aaef87913a775a19b89b21f35fc2b9cfbfdb3f135013e027c3eefd854963700cd91ef08be0c1f3976cf597f1c2f358430a4cb565
-  data.tar.gz: 543c853757732ced23ebdbf4d5caa1ef09a91ca9ae4f20b36d75d32cb383d5d56c50548ea873199fcb2324a44dd77ee9c8ac8efe51568a4c94d421b5e45e53d9
+  metadata.gz: f75b481668bc142794f1f683c1e8eac55d6b26e333c5a7ecdd91afe748a454d5dbc47e81b99c61cc8c910421f84d1789d8ba38370ac1c4867a98ce4bfea1582d
+  data.tar.gz: afd865e08ba92c6bf4f231ddf8025af16a4e3fd623e6a0dce2d5e49975020b38c316b36d976e5a3e7875ca4d3b66d0bfdf4c7d27fff218527b20ebeed5f235c4

data/lib/legion/extensions/eval/actors/online.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/actors/subscription' unless defined?(Legion::Extensions::Actors::Subscription)
+
+module Legion
+  module Extensions
+    module Eval
+      module Actor
+        class Online < Legion::Extensions::Actors::Subscription
+          EXCHANGE = 'llm.response'
+          QUEUE    = 'eval.online'
+
+          def runner_class
+            Legion::Extensions::Eval::Runners::Online
+          end
+
+          def runner_function
+            'evaluate_response'
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+
+          def use_runner?
+            false
+          end
+
+          def enabled?
+            return false unless defined?(Legion::Transport)
+            return false unless defined?(Legion::Extensions::Eval::Runners::Online)
+
+            online_enabled?
+          rescue StandardError
+            false
+          end
+
+          private
+
+          def online_enabled?
+            return true unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :enabled) != false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/eval/client.rb

@@ -7,6 +7,7 @@ module Legion
         include Runners::Evaluation
         include Runners::Annotation
         include Runners::AgenticReview
+        include Runners::Online
 
         def initialize(db: nil, **opts)
           @db = db

data/lib/legion/extensions/eval/runners/online.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Eval
+      module Runners
+        module Online
+          def evaluate_response(response:, evaluators: nil, sample_rate: 1.0, **)
+            evaluator_names = evaluators || configured_evaluators
+            effective_rate  = sample_rate || configured_sample_rate
+
+            return { evaluated: false, reason: :sampled_out, sampled: false } unless rand <= effective_rate
+
+            scores = {}
+            evaluator_names.each do |name|
+              scores[name.to_sym] = run_single_evaluator(name, response)
+            end
+
+            { evaluated: true, scores: scores, sampled: true }
+          rescue StandardError => e
+            Legion::Logging.warn("lex-eval online: evaluate_response failed: #{e.message}") if defined?(Legion::Logging)
+            { evaluated: false, reason: :error, error: e.message, sampled: true }
+          end
+
+          private
+
+          def run_single_evaluator(name, response)
+            loader = Helpers::TemplateLoader.new
+            config = loader.load_template(name.to_s) || {}
+            result = run_evaluation(
+              evaluator_name:   name,
+              evaluator_config: config,
+              inputs:           [{ input: response[:input] || '', output: response[:output] || '' }]
+            )
+            result.dig(:summary, :avg_score)
+          rescue StandardError => e
+            Legion::Logging.warn("lex-eval online: evaluator #{name} failed: #{e.message}") if defined?(Legion::Logging)
+            nil
+          end
+
+          def configured_evaluators
+            return %w[toxicity] unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :evaluators) || %w[toxicity]
+          end
+
+          def configured_sample_rate
+            return 1.0 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :sample_rate) || 1.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.2.1'
+      VERSION = '0.2.2'
     end
   end
 end

data/lib/legion/extensions/eval.rb

@@ -10,6 +10,7 @@ require_relative 'eval/helpers/guardrails'
 require_relative 'eval/runners/evaluation'
 require_relative 'eval/runners/annotation'
 require_relative 'eval/runners/agentic_review'
+require_relative 'eval/runners/online'
 require_relative 'eval/client'
 
 module Legion

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Matthew Iverson
@@ -19,6 +19,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/legion/extensions/eval.rb
+- lib/legion/extensions/eval/actors/online.rb
 - lib/legion/extensions/eval/client.rb
 - lib/legion/extensions/eval/evaluators/base.rb
 - lib/legion/extensions/eval/evaluators/code_evaluator.rb
@@ -32,6 +33,7 @@ files:
 - lib/legion/extensions/eval/runners/agentic_review.rb
 - lib/legion/extensions/eval/runners/annotation.rb
 - lib/legion/extensions/eval/runners/evaluation.rb
+- lib/legion/extensions/eval/runners/online.rb
 - lib/legion/extensions/eval/templates/code_generation.yml
 - lib/legion/extensions/eval/templates/code_readability.yml
 - lib/legion/extensions/eval/templates/faithfulness.yml