lex-eval 0.3.11 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ba8f0431d907def112bdd8f02a0685002fbfc75754bb658d620ea4889b03c70
-  data.tar.gz: b38fcf01598441f544cf8721d9109010983b357be7085e4e0196edd9872cd16a
+  metadata.gz: bcf09bf5fdfb1a7c80e050e440c874f6e099ddd41ac0f85155733d1c4aa80bed
+  data.tar.gz: f1291ca88a8a4f7cf30abce7616bf8ab4145d1db0fe6ed6c0112e09482bda77b
 SHA512:
-  metadata.gz: 678d325ad2c47aa74abfe3a8ec4f8c7eb63bb14beec4e8911c7912963ded1cc966dbe57dfe50b721e54cdd85df1ef67e6c04af73004c76f560e9b5b45c0ea132
-  data.tar.gz: abd3cab893f187e0c51929cb9f91fa7f3806d250a339c6c137e6073589d0d6e0a18e9560dcdc9645b39f07c9471e9b93ee5ae0eaf3b74f421501383ba3951078
+  metadata.gz: 1df90ff664f437acbfa6eb6e1b7dd5efc7620a8b4ee73b8f11a4fb5b35f8b2ab7324d887186abf56c929a84799e68eba3a30a375b05760048fab6eb6f1078b2f
+  data.tar.gz: 1887981dc5dc141deb187f1d8ed7e2c5075c911d87b7a367773b93fd96ea6b52c6564b44a90ee8c32a1e9e6670c5ba381b371a8440fdad39b8ea9fda2d42cb9b

data/lib/legion/extensions/eval/actors/code_review_subscriber.rb

@@ -11,6 +11,7 @@ module Legion
 
           def runner_class = self.class
           def runner_function = 'action'
+          def check_subtask? = true
 
           def action(payload)
             code = payload[:runner_code] || payload[:code]
@@ -30,17 +31,13 @@ module Legion
             result
           rescue StandardError => e
             log.warn("CodeReviewSubscriber failed: #{e.message}")
-            { passed: false, verdict: :reject, error: e.message }
+            { passed: false, verdict: 'reject', error: e.message }
           end
 
           private
 
           def log
-            return Legion::Logging if defined?(Legion::Logging)
-
-            @log ||= Object.new.tap do |nl|
-              %i[debug info warn error fatal].each { |m| nl.define_singleton_method(m) { |*| nil } }
-            end
+            Legion::Logging
           end
         end
       end

data/lib/legion/extensions/eval/actors/online.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'legion/extensions/actors/subscription' unless defined?(Legion::Extensions::Actors::Subscription)
+require 'legion/extensions/actors/subscription'
 
 module Legion
   module Extensions
@@ -30,12 +30,12 @@ module Legion
             false
           end
 
-          def enabled?
-            return false unless defined?(Legion::Transport)
+          def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
+            return false unless Legion.const_defined?(:Transport, false)
             return false unless defined?(Legion::Extensions::Eval::Runners::Online)
 
             online_enabled?
-          rescue StandardError
+          rescue StandardError => _e
             false
           end
 

data/lib/legion/extensions/eval/evaluators/code_evaluator.rb

@@ -31,7 +31,7 @@ module Legion
           def valid_json?(str)
             ::JSON.parse(str)
             true
-          rescue ::JSON::ParserError
+          rescue ::JSON::ParserError => _e
             false
           end
         end

data/lib/legion/extensions/eval/evaluators/llm_judge.rb

@@ -37,24 +37,24 @@ module Legion
           def evaluate_impl(input:, output:, expected:)
             prompt = render_template(input: input, output: output, expected: expected)
             evaluate_structured(prompt)
-          rescue StandardError
+          rescue StandardError => _e
             evaluate_regex_fallback(prompt)
           end
 
           def evaluate_structured(prompt)
             return evaluate_regex_fallback(prompt) unless structured_available?
 
-            result = Legion::LLM.structured(message: prompt, schema: JUDGE_SCHEMA,
+            result = Legion::LLM.structured(message: prompt, schema: JUDGE_SCHEMA, # rubocop:disable Legion/HelperMigration/DirectLlm
                                             intent: { capability: :reasoning },
                                             caller: { extension: 'lex-eval', operation: 'judge' })
             { score: result[:score], passed: result[:passed],
               explanation: result[:explanation], evidence: result[:evidence] || [] }
-          rescue StandardError
+          rescue StandardError => _e
             evaluate_regex_fallback(prompt)
           end
 
           def evaluate_regex_fallback(prompt)
-            response = Legion::LLM.chat(message: prompt, intent: { capability: :reasoning },
+            response = Legion::LLM.chat(message: prompt, intent: { capability: :reasoning }, # rubocop:disable Legion/HelperMigration/DirectLlm
                                         caller: { extension: 'lex-eval', operation: 'judge' })
             score = extract_score(response.content)
             { score: score, explanation: response.content, passed: score >= threshold, evidence: [] }

data/lib/legion/extensions/eval/helpers/guardrails.rb

@@ -79,11 +79,7 @@ module Legion
             end
 
             def log
-              return Legion::Logging if defined?(Legion::Logging)
-
-              @log ||= Object.new.tap do |nl|
-                %i[debug info warn error fatal].each { |m| nl.define_singleton_method(m) { |*| nil } }
-              end
+              Legion::Logging
             end
           end
         end

data/lib/legion/extensions/eval/runners/agentic_review.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Eval
       module Runners
-        module AgenticReview
+        module AgenticReview # rubocop:disable Legion/Extension/RunnerIncludeHelpers
           REVIEW_SCHEMA = {
             type:       :object,
             properties: {
@@ -23,7 +23,7 @@ module Legion
             required:   %i[confidence recommendation explanation]
           }.freeze
 
-          def review_output(input:, output:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
+          def review_output(input:, output:, review_prompt: nil, model: nil, provider: nil, **)
             prompt = build_review_message(review_prompt || default_review_prompt, input, output)
             llm_kwargs = {
               message: prompt, schema: REVIEW_SCHEMA,
@@ -32,14 +32,14 @@ module Legion
             }
             llm_kwargs[:model] = model if model
             llm_kwargs[:provider] = provider if provider
-            Legion::LLM.structured(**llm_kwargs)
+            Legion::LLM.structured(**llm_kwargs) # rubocop:disable Legion/HelperMigration/DirectLlm
           rescue StandardError => e
-            log.warn(e.message) if respond_to?(:log, true)
+            log.warn(e.message) if respond_to?(:log, true) # rubocop:disable Legion/HelperMigration/LoggingGuard
             { confidence: 0.0, recommendation: 'reject',
               issues: [], explanation: "review error: #{e.message}" }
           end
 
-          def review_with_escalation(input:, output:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
+          def review_with_escalation(input:, output:, review_prompt: nil, model: nil, provider: nil, **)
             review = review_output(input: input, output: output, review_prompt: review_prompt,
                                    model: model, provider: provider)
             action, priority = determine_escalation(review[:confidence])
@@ -49,7 +49,7 @@ module Legion
             review.merge(action: action, escalated: true, priority: priority)
           end
 
-          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
+          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, model: nil, provider: nil, **)
             review_a = review_output(input: input, output: output_a, review_prompt: review_prompt,
                                      model: model, provider: provider)
             review_b = review_output(input: input, output: output_b, review_prompt: review_prompt,
@@ -73,7 +73,7 @@ module Legion
               review_a: review_a,
               review_b: review_b }
           rescue StandardError => e
-            log.warn(e.message) if respond_to?(:log, true)
+            log.warn(e.message) if respond_to?(:log, true) # rubocop:disable Legion/HelperMigration/LoggingGuard
             { reviewed: false, reason: "experiment error: #{e.message}" }
           end
 

data/lib/legion/extensions/eval/runners/annotation.rb

@@ -5,6 +5,8 @@ module Legion
     module Eval
       module Runners
         module Annotation
+          extend self
+
           def create_queue(name:, **opts)
             db[:annotation_queues].insert(
               name:                name,
@@ -15,7 +17,7 @@ module Legion
               created_at:          Time.now.utc
             )
             { created: true, name: name }
-          rescue Sequel::UniqueConstraintViolation
+          rescue Sequel::UniqueConstraintViolation => _e
             { error: 'already_exists', name: name }
           end
 

data/lib/legion/extensions/eval/runners/code_review.rb

@@ -13,7 +13,7 @@ module Legion
 
           SPEC_TIMEOUT = 30
 
-          def review_generated(code:, spec_code:, context:, review_k: nil, review_models: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+          def review_generated(code:, spec_code:, context: {}, review_k: nil, review_models: nil, **extra) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
             settings = validation_settings
             stages = {}
             issues = []
@@ -22,8 +22,9 @@ module Legion
             if settings[:syntax_check] != false
               stages[:syntax] = check_syntax(code, spec_code)
               unless stages[:syntax][:passed]
-                return build_result(passed: false, verdict: :reject, stages: stages,
-                                    issues: stages[:syntax][:errors], confidence: 0.0)
+                return build_result(passed: false, verdict: 'reject', stages: stages,
+                                    issues: stages[:syntax][:errors], confidence: 0.0,
+                                    code: code, spec_code: spec_code, extra: extra)
               end
             end
 
@@ -31,7 +32,8 @@ module Legion
             stages[:security] = check_security(code)
             unless stages[:security][:passed]
               issues.concat(stages[:security][:flagged].map { |f| "security: #{f[:pattern]} on line #{f[:line]}" })
-              return build_result(passed: false, verdict: :reject, stages: stages, issues: issues, confidence: 0.0)
+              return build_result(passed: false, verdict: 'reject', stages: stages, issues: issues, confidence: 0.0,
+                                  code: code, spec_code: spec_code, extra: extra)
             end
 
             # Stage 3: Spec execution (optional)
@@ -39,7 +41,8 @@ module Legion
               stages[:specs] = run_specs(code, spec_code)
               unless stages[:specs][:passed]
                 issues << "specs failed: #{stages[:specs][:output]}"
-                return build_result(passed: false, verdict: :revise, stages: stages, issues: issues, confidence: 0.2)
+                return build_result(passed: false, verdict: 'revise', stages: stages, issues: issues, confidence: 0.2,
+                                    code: code, spec_code: spec_code, extra: extra)
               end
             end
 
@@ -65,9 +68,10 @@ module Legion
             end
 
             confidence = calculate_confidence(stages)
-            verdict = confidence >= 0.5 ? :approve : :revise
+            verdict = confidence >= 0.5 ? 'approve' : 'revise'
 
-            build_result(passed: true, verdict: verdict, stages: stages, issues: issues, confidence: confidence)
+            build_result(passed: true, verdict: verdict, stages: stages, issues: issues, confidence: confidence,
+                         code: code, spec_code: spec_code, extra: extra)
           end
 
           private
@@ -108,7 +112,7 @@ module Legion
             false
           end
 
-          def build_model_assignments(count, models) # rubocop:disable Metrics/PerceivedComplexity
+          def build_model_assignments(count, models)
             return Array.new(count) { nil } if models.nil? || models.empty?
 
             available = models.select do |spec|
@@ -128,7 +132,7 @@ module Legion
             Array.new(count) { |i| available[i % available.size] }
           end
 
-          def adversarial_llm_review(code, context, count:, models: []) # rubocop:disable Metrics/PerceivedComplexity
+          def adversarial_llm_review(code, context, count:, models: [])
             assignments = build_model_assignments(count, models)
 
             reviews = assignments.map { |spec| llm_review(code, context, model_spec: spec) }
@@ -201,7 +205,7 @@ module Legion
             { passed: false, output: '', errors: e.message, exit_code: -1 }
           end
 
-          def llm_review(code, context, model_spec: nil) # rubocop:disable Metrics/PerceivedComplexity
+          def llm_review(code, context, model_spec: nil)
             return { passed: true, issues: [], confidence: 0.5 } unless defined?(Runners::AgenticReview)
 
             extra_kwargs = {}
@@ -272,7 +276,7 @@ module Legion
             scores.sum / scores.size
           end
 
-          def stage_scores(stages) # rubocop:disable Metrics/PerceivedComplexity
+          def stage_scores(stages)
             scores = []
             scores << (stage_passed?(stages[:syntax]) ? 1.0 : 0.0) if stages[:syntax]
             scores << (stage_passed?(stages[:security]) ? 1.0 : 0.0) if stages[:security]
@@ -282,8 +286,12 @@ module Legion
             scores
           end
 
-          def build_result(passed:, verdict:, stages:, issues:, confidence:)
-            { passed: passed, verdict: verdict, confidence: confidence, stages: stages, issues: issues }
+          def build_result(passed:, verdict:, stages:, issues:, confidence:, code: nil, spec_code: nil, extra: {})
+            result = { passed: passed, verdict: verdict, confidence: confidence, stages: stages, issues: issues }
+            result[:code] = code
+            result[:spec_code] = spec_code
+            extra.each { |k, v| result[k] = v unless result.key?(k) }
+            result
           end
         end
       end

data/lib/legion/extensions/eval/runners/evaluation.rb

@@ -5,6 +5,8 @@ module Legion
     module Eval
       module Runners
         module Evaluation
+          extend self # rubocop:disable Style/ModuleFunction
+
           def run_evaluation(evaluator_name:, evaluator_config: {}, inputs: [], **)
             evaluator = build_evaluator(evaluator_name, evaluator_config)
             results = inputs.map.with_index do |row, idx|

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.3.11'
+      VERSION = '0.3.13'
     end
   end
 end

data/lib/legion/extensions/eval.rb

@@ -22,7 +22,7 @@ if defined?(Legion::Transport::Exchange)
   require_relative 'eval/transport/messages/code_review_completed'
 end
 
-require_relative 'eval/actors/code_review_subscriber' if defined?(Legion::Extensions::Actors::Subscription)
+require_relative 'eval/actors/code_review_subscriber'
 
 module Legion
   module Extensions

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.13
 platform: ruby
 authors:
 - Matthew Iverson