lex-eval 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dd068d711cd3cc0c70d64f8c066e1bb03e929bc034073600a8e3946c7c65a77
-  data.tar.gz: 6103505a44655acc55a78ac3677b2d8fef300e395d33acb47e5a545cd0f7e8e3
+  metadata.gz: 42e93ebdc972bbb947ca388705947747e0dc4050e7cb5483efc57471adfbca29
+  data.tar.gz: f3a45e7d94bd7c4670f92f11a6e838e9a0913fd8823726436efa889c017afea3
 SHA512:
-  metadata.gz: 4b0ef19e8406c5eaf2914b22aaef87913a775a19b89b21f35fc2b9cfbfdb3f135013e027c3eefd854963700cd91ef08be0c1f3976cf597f1c2f358430a4cb565
-  data.tar.gz: 543c853757732ced23ebdbf4d5caa1ef09a91ca9ae4f20b36d75d32cb383d5d56c50548ea873199fcb2324a44dd77ee9c8ac8efe51568a4c94d421b5e45e53d9
+  metadata.gz: 9597e0c4cf93d703c40d83a80107a1c2271022710dadfc666ea3ce510d327eefceb33005e456b0af1064d77c5d7d88502d3476cf7554382fc0b7d18e07421539
+  data.tar.gz: b8e858dbd043dfd71fe5d964c608d438ad34f56ba3da487fd3a58f8fd76ffd244f01266a0c72c892c1257ac87f0041a1a7e7aaf1fb09f6ff2ef48e53ab0c13f7

data/lib/legion/extensions/eval/actors/online.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/actors/subscription' unless defined?(Legion::Extensions::Actors::Subscription)
+
+module Legion
+  module Extensions
+    module Eval
+      module Actor
+        class Online < Legion::Extensions::Actors::Subscription
+          EXCHANGE = 'llm.response'
+          QUEUE    = 'eval.online'
+
+          def runner_class
+            Legion::Extensions::Eval::Runners::Online
+          end
+
+          def runner_function
+            'evaluate_response'
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+
+          def use_runner?
+            false
+          end
+
+          def enabled?
+            return false unless defined?(Legion::Transport)
+            return false unless defined?(Legion::Extensions::Eval::Runners::Online)
+
+            online_enabled?
+          rescue StandardError
+            false
+          end
+
+          private
+
+          def online_enabled?
+            return true unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :enabled) != false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/eval/client.rb

@@ -7,6 +7,7 @@ module Legion
         include Runners::Evaluation
         include Runners::Annotation
         include Runners::AgenticReview
+        include Runners::Online
 
         def initialize(db: nil, **opts)
           @db = db

data/lib/legion/extensions/eval/runners/online.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Eval
+      module Runners
+        module Online
+          include Legion::Extensions::Helpers::Lex if defined?(Legion::Extensions::Helpers::Lex)
+          def evaluate_response(response:, evaluators: nil, sample_rate: 1.0, **)
+            evaluator_names = evaluators || configured_evaluators
+            effective_rate  = sample_rate || configured_sample_rate
+
+            return { evaluated: false, reason: :sampled_out, sampled: false } unless rand <= effective_rate
+
+            scores = {}
+            evaluator_names.each do |name|
+              scores[name.to_sym] = run_single_evaluator(name, response)
+            end
+
+            { evaluated: true, scores: scores, sampled: true }
+          rescue StandardError => e
+            log.warn("lex-eval online: evaluate_response failed: #{e.message}")
+            { evaluated: false, reason: :error, error: e.message, sampled: true }
+          end
+
+          private
+
+          def run_single_evaluator(name, response)
+            loader = Helpers::TemplateLoader.new
+            config = loader.load_template(name.to_s) || {}
+            result = run_evaluation(
+              evaluator_name:   name,
+              evaluator_config: config,
+              inputs:           [{ input: response[:input] || '', output: response[:output] || '' }]
+            )
+            result.dig(:summary, :avg_score)
+          rescue StandardError => e
+            log.warn("lex-eval online: evaluator #{name} failed: #{e.message}")
+            nil
+          end
+
+          def configured_evaluators
+            return %w[toxicity] unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :evaluators) || %w[toxicity]
+          end
+
+          def configured_sample_rate
+            return 1.0 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:eval, :online, :sample_rate) || 1.0
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.2.1'
+      VERSION = '0.2.3'
     end
   end
 end

data/lib/legion/extensions/eval.rb

@@ -10,6 +10,7 @@ require_relative 'eval/helpers/guardrails'
 require_relative 'eval/runners/evaluation'
 require_relative 'eval/runners/annotation'
 require_relative 'eval/runners/agentic_review'
+require_relative 'eval/runners/online'
 require_relative 'eval/client'
 
 module Legion

metadata

@@ -1,14 +1,112 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Matthew Iverson
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: legion-cache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.11
+- !ruby/object:Gem::Dependency
+  name: legion-crypt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.9
+- !ruby/object:Gem::Dependency
+  name: legion-data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.17
+- !ruby/object:Gem::Dependency
+  name: legion-json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: legion-logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: legion-settings
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.14
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.14
+- !ruby/object:Gem::Dependency
+  name: legion-transport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.9
 description: Provides LLM-as-judge and code-based evaluators for scoring LLM outputs,
   with built-in templates for hallucination, relevance, and toxicity detection.
 email:
@@ -19,6 +117,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/legion/extensions/eval.rb
+- lib/legion/extensions/eval/actors/online.rb
 - lib/legion/extensions/eval/client.rb
 - lib/legion/extensions/eval/evaluators/base.rb
 - lib/legion/extensions/eval/evaluators/code_evaluator.rb
@@ -32,6 +131,7 @@ files:
 - lib/legion/extensions/eval/runners/agentic_review.rb
 - lib/legion/extensions/eval/runners/annotation.rb
 - lib/legion/extensions/eval/runners/evaluation.rb
+- lib/legion/extensions/eval/runners/online.rb
 - lib/legion/extensions/eval/templates/code_generation.yml
 - lib/legion/extensions/eval/templates/code_readability.yml
 - lib/legion/extensions/eval/templates/faithfulness.yml