lex-eval 0.3.10 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3214b4f11d574772ad1323a0b37816d45c29ac61e5713f2ee82776ac8906ec09
-  data.tar.gz: b3bd8441ea3a5eda81b9f402e3d7e44e803504002ada86695c41b885f76d624b
+  metadata.gz: bcf09bf5fdfb1a7c80e050e440c874f6e099ddd41ac0f85155733d1c4aa80bed
+  data.tar.gz: f1291ca88a8a4f7cf30abce7616bf8ab4145d1db0fe6ed6c0112e09482bda77b
 SHA512:
-  metadata.gz: 97601e029b078a0fbf7c1606968e90f5e5f308be477f543e793ff8533eb3ff6ff62716614388eda67416cb671b45e3a07d3d77e58fdf80484944c24bd866cc5f
-  data.tar.gz: c20ab43b7eed22645560f0022094cb8c2b7c9ae0044c452aa57fcdb524e6833bcc30e07b801ec58b52175e2b12871b44e7dace59dfa20499e0a9e6b0529a6ea4
+  metadata.gz: 1df90ff664f437acbfa6eb6e1b7dd5efc7620a8b4ee73b8f11a4fb5b35f8b2ab7324d887186abf56c929a84799e68eba3a30a375b05760048fab6eb6f1078b2f
+  data.tar.gz: 1887981dc5dc141deb187f1d8ed7e2c5075c911d87b7a367773b93fd96ea6b52c6564b44a90ee8c32a1e9e6670c5ba381b371a8440fdad39b8ea9fda2d42cb9b

data/lib/legion/extensions/eval/actors/code_review_subscriber.rb

@@ -11,6 +11,7 @@ module Legion
 
           def runner_class = self.class
           def runner_function = 'action'
+          def check_subtask? = true
 
           def action(payload)
             code = payload[:runner_code] || payload[:code]
@@ -30,17 +31,13 @@ module Legion
             result
           rescue StandardError => e
             log.warn("CodeReviewSubscriber failed: #{e.message}")
-            { passed: false, verdict: :reject, error: e.message }
+            { passed: false, verdict: 'reject', error: e.message }
           end
 
           private
 
           def log
-            return Legion::Logging if defined?(Legion::Logging)
-
-            @log ||= Object.new.tap do |nl|
-              %i[debug info warn error fatal].each { |m| nl.define_singleton_method(m) { |*| nil } }
-            end
+            Legion::Logging
           end
         end
       end

data/lib/legion/extensions/eval/actors/online.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'legion/extensions/actors/subscription' unless defined?(Legion::Extensions::Actors::Subscription)
+require 'legion/extensions/actors/subscription'
 
 module Legion
   module Extensions
@@ -30,12 +30,12 @@ module Legion
             false
           end
 
-          def enabled?
-            return false unless defined?(Legion::Transport)
+          def enabled? # rubocop:disable Legion/Extension/ActorEnabledSideEffects
+            return false unless Legion.const_defined?(:Transport, false)
             return false unless defined?(Legion::Extensions::Eval::Runners::Online)
 
             online_enabled?
-          rescue StandardError
+          rescue StandardError => _e
             false
           end
 

data/lib/legion/extensions/eval/evaluators/code_evaluator.rb

@@ -31,7 +31,7 @@ module Legion
           def valid_json?(str)
             ::JSON.parse(str)
             true
-          rescue ::JSON::ParserError
+          rescue ::JSON::ParserError => _e
             false
           end
         end

data/lib/legion/extensions/eval/evaluators/llm_judge.rb

@@ -37,24 +37,24 @@ module Legion
           def evaluate_impl(input:, output:, expected:)
             prompt = render_template(input: input, output: output, expected: expected)
             evaluate_structured(prompt)
-          rescue StandardError
+          rescue StandardError => _e
             evaluate_regex_fallback(prompt)
           end
 
           def evaluate_structured(prompt)
             return evaluate_regex_fallback(prompt) unless structured_available?
 
-            result = Legion::LLM.structured(message: prompt, schema: JUDGE_SCHEMA,
+            result = Legion::LLM.structured(message: prompt, schema: JUDGE_SCHEMA, # rubocop:disable Legion/HelperMigration/DirectLlm
                                             intent: { capability: :reasoning },
                                             caller: { extension: 'lex-eval', operation: 'judge' })
             { score: result[:score], passed: result[:passed],
               explanation: result[:explanation], evidence: result[:evidence] || [] }
-          rescue StandardError
+          rescue StandardError => _e
             evaluate_regex_fallback(prompt)
           end
 
           def evaluate_regex_fallback(prompt)
-            response = Legion::LLM.chat(message: prompt, intent: { capability: :reasoning },
+            response = Legion::LLM.chat(message: prompt, intent: { capability: :reasoning }, # rubocop:disable Legion/HelperMigration/DirectLlm
                                         caller: { extension: 'lex-eval', operation: 'judge' })
             score = extract_score(response.content)
             { score: score, explanation: response.content, passed: score >= threshold, evidence: [] }

data/lib/legion/extensions/eval/helpers/guardrails.rb

@@ -79,11 +79,7 @@ module Legion
             end
 
             def log
-              return Legion::Logging if defined?(Legion::Logging)
-
-              @log ||= Object.new.tap do |nl|
-                %i[debug info warn error fatal].each { |m| nl.define_singleton_method(m) { |*| nil } }
-              end
+              Legion::Logging
             end
           end
         end

data/lib/legion/extensions/eval/runners/agentic_review.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Eval
       module Runners
-        module AgenticReview
+        module AgenticReview # rubocop:disable Legion/Extension/RunnerIncludeHelpers
           REVIEW_SCHEMA = {
             type:       :object,
             properties: {
@@ -23,18 +23,25 @@ module Legion
             required:   %i[confidence recommendation explanation]
           }.freeze
 
-          def review_output(input:, output:, review_prompt: nil, **)
+          def review_output(input:, output:, review_prompt: nil, model: nil, provider: nil, **)
             prompt = build_review_message(review_prompt || default_review_prompt, input, output)
-            Legion::LLM.structured(message: prompt, schema: REVIEW_SCHEMA,
-                                   intent: { capability: :reasoning },
-                                   caller: { extension: 'lex-eval', operation: 'agentic_review' })
+            llm_kwargs = {
+              message: prompt, schema: REVIEW_SCHEMA,
+              intent: { capability: :reasoning },
+              caller: { extension: 'lex-eval', operation: 'agentic_review' }
+            }
+            llm_kwargs[:model] = model if model
+            llm_kwargs[:provider] = provider if provider
+            Legion::LLM.structured(**llm_kwargs) # rubocop:disable Legion/HelperMigration/DirectLlm
           rescue StandardError => e
+            log.warn(e.message) if respond_to?(:log, true) # rubocop:disable Legion/HelperMigration/LoggingGuard
             { confidence: 0.0, recommendation: 'reject',
               issues: [], explanation: "review error: #{e.message}" }
           end
 
-          def review_with_escalation(input:, output:, review_prompt: nil, **)
-            review = review_output(input: input, output: output, review_prompt: review_prompt)
+          def review_with_escalation(input:, output:, review_prompt: nil, model: nil, provider: nil, **)
+            review = review_output(input: input, output: output, review_prompt: review_prompt,
+                                   model: model, provider: provider)
             action, priority = determine_escalation(review[:confidence])
 
             return review.merge(action: :auto_approve, escalated: false) if action == :auto_approve
@@ -42,9 +49,11 @@ module Legion
             review.merge(action: action, escalated: true, priority: priority)
           end
 
-          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, **)
-            review_a = review_output(input: input, output: output_a, review_prompt: review_prompt)
-            review_b = review_output(input: input, output: output_b, review_prompt: review_prompt)
+          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, model: nil, provider: nil, **)
+            review_a = review_output(input: input, output: output_a, review_prompt: review_prompt,
+                                     model: model, provider: provider)
+            review_b = review_output(input: input, output: output_b, review_prompt: review_prompt,
+                                     model: model, provider: provider)
 
             conf_a = review_a[:confidence] || 0.0
             conf_b = review_b[:confidence] || 0.0
@@ -64,6 +73,7 @@ module Legion
               review_a: review_a,
               review_b: review_b }
           rescue StandardError => e
+            log.warn(e.message) if respond_to?(:log, true) # rubocop:disable Legion/HelperMigration/LoggingGuard
             { reviewed: false, reason: "experiment error: #{e.message}" }
           end
 

data/lib/legion/extensions/eval/runners/annotation.rb

@@ -5,6 +5,8 @@ module Legion
     module Eval
       module Runners
         module Annotation
+          extend self
+
           def create_queue(name:, **opts)
             db[:annotation_queues].insert(
               name:                name,
@@ -15,7 +17,7 @@ module Legion
               created_at:          Time.now.utc
             )
             { created: true, name: name }
-          rescue Sequel::UniqueConstraintViolation
+          rescue Sequel::UniqueConstraintViolation => _e
             { error: 'already_exists', name: name }
           end
 

data/lib/legion/extensions/eval/runners/code_review.rb

@@ -13,7 +13,7 @@ module Legion
 
           SPEC_TIMEOUT = 30
 
-          def review_generated(code:, spec_code:, context:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+          def review_generated(code:, spec_code:, context: {}, review_k: nil, review_models: nil, **extra) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
             settings = validation_settings
             stages = {}
             issues = []
@@ -22,8 +22,9 @@ module Legion
             if settings[:syntax_check] != false
               stages[:syntax] = check_syntax(code, spec_code)
               unless stages[:syntax][:passed]
-                return build_result(passed: false, verdict: :reject, stages: stages,
-                                    issues: stages[:syntax][:errors], confidence: 0.0)
+                return build_result(passed: false, verdict: 'reject', stages: stages,
+                                    issues: stages[:syntax][:errors], confidence: 0.0,
+                                    code: code, spec_code: spec_code, extra: extra)
               end
             end
 
@@ -31,7 +32,8 @@ module Legion
             stages[:security] = check_security(code)
             unless stages[:security][:passed]
               issues.concat(stages[:security][:flagged].map { |f| "security: #{f[:pattern]} on line #{f[:line]}" })
-              return build_result(passed: false, verdict: :reject, stages: stages, issues: issues, confidence: 0.0)
+              return build_result(passed: false, verdict: 'reject', stages: stages, issues: issues, confidence: 0.0,
+                                  code: code, spec_code: spec_code, extra: extra)
             end
 
             # Stage 3: Spec execution (optional)
@@ -39,13 +41,20 @@ module Legion
               stages[:specs] = run_specs(code, spec_code)
               unless stages[:specs][:passed]
                 issues << "specs failed: #{stages[:specs][:output]}"
-                return build_result(passed: false, verdict: :revise, stages: stages, issues: issues, confidence: 0.2)
+                return build_result(passed: false, verdict: 'revise', stages: stages, issues: issues, confidence: 0.2,
+                                    code: code, spec_code: spec_code, extra: extra)
               end
             end
 
             # Stage 4: LLM review (optional)
             if settings[:llm_review] && llm_available?
-              stages[:llm_review] = llm_review(code, context)
+              k = review_k || default_review_k
+              models = review_models || default_review_models
+              stages[:llm_review] = if k > 1
+                                      adversarial_llm_review(code, context, count: k, models: models)
+                                    else
+                                      llm_review(code, context, model_spec: build_model_assignments(1, models)&.first)
+                                    end
               issues.concat(stages[:llm_review][:issues] || [])
             end
 
@@ -59,9 +68,10 @@ module Legion
             end
 
             confidence = calculate_confidence(stages)
-            verdict = confidence >= 0.5 ? :approve : :revise
+            verdict = confidence >= 0.5 ? 'approve' : 'revise'
 
-            build_result(passed: true, verdict: verdict, stages: stages, issues: issues, confidence: confidence)
+            build_result(passed: true, verdict: verdict, stages: stages, issues: issues, confidence: confidence,
+                         code: code, spec_code: spec_code, extra: extra)
           end
 
           private
@@ -75,6 +85,80 @@ module Legion
             {}
           end
 
+          def default_review_k
+            return 1 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:codegen, :self_generate, :validation, :review_k) || 1
+          rescue StandardError => e
+            log.warn(e.message)
+            1
+          end
+
+          def default_review_models
+            return [] unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:codegen, :self_generate, :validation, :review_models) || []
+          rescue StandardError => e
+            log.warn(e.message)
+            []
+          end
+
+          def provider_available?(provider_sym)
+            return false unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:llm, :providers, provider_sym, :enabled) == true
+          rescue StandardError => e
+            log.warn(e.message)
+            false
+          end
+
+          def build_model_assignments(count, models)
+            return Array.new(count) { nil } if models.nil? || models.empty?
+
+            available = models.select do |spec|
+              next false unless spec.is_a?(Hash)
+
+              provider_sym = spec[:provider]&.to_sym
+              if provider_sym && !provider_available?(provider_sym)
+                log.warn("review provider #{provider_sym} not available, skipping")
+                false
+              else
+                true
+              end
+            end
+
+            return Array.new(count) { nil } if available.empty?
+
+            Array.new(count) { |i| available[i % available.size] }
+          end
+
+          def adversarial_llm_review(code, context, count:, models: [])
+            assignments = build_model_assignments(count, models)
+
+            reviews = assignments.map { |spec| llm_review(code, context, model_spec: spec) }
+
+            approvals = reviews.count { |r| r[:confidence] >= 0.5 }
+            rejections = count - approvals
+            all_issues = reviews.flat_map { |r| r[:issues] || [] }.uniq
+
+            avg_confidence = reviews.sum { |r| r[:confidence] || 0.0 } / reviews.size
+
+            {
+              passed:     approvals > rejections,
+              issues:     all_issues,
+              confidence: avg_confidence,
+              k:          count,
+              approvals:  approvals,
+              rejections: rejections,
+              reviews:    reviews
+            }
+          rescue StandardError => e
+            log.warn("adversarial review failed: #{e.message}")
+            fallback = llm_review(code, context)
+            fallback.merge(k: count, approvals: (fallback[:passed] ? 1 : 0),
+                           rejections: (fallback[:passed] ? 0 : 1), reviews: [fallback])
+          end
+
           def check_syntax(code, spec_code)
             errors = []
             begin
@@ -121,19 +205,28 @@ module Legion
             { passed: false, output: '', errors: e.message, exit_code: -1 }
           end
 
-          def llm_review(code, context)
+          def llm_review(code, context, model_spec: nil)
             return { passed: true, issues: [], confidence: 0.5 } unless defined?(Runners::AgenticReview)
 
+            extra_kwargs = {}
+            if model_spec
+              extra_kwargs[:model] = model_spec[:model] if model_spec[:model]
+              extra_kwargs[:provider] = model_spec[:provider] if model_spec[:provider]
+            end
+
             result = Runners::AgenticReview.review_output(
               input:         context,
               output:        code,
-              review_prompt: 'Review this generated Ruby code for correctness, safety, and Legion conventions.'
+              review_prompt: 'Review this generated Ruby code for correctness, safety, and Legion conventions.',
+              **extra_kwargs
             )
 
             {
-              passed:     result[:reviewed] != false,
+              passed:     result[:recommendation] == 'approve',
               issues:     result[:issues] || [],
-              confidence: result[:confidence] || 0.5
+              confidence: result[:confidence] || 0.5,
+              provider:   model_spec&.dig(:provider),
+              model:      model_spec&.dig(:model)
             }
           rescue StandardError => e
             log.warn("llm review failed: #{e.message}")
@@ -183,7 +276,7 @@ module Legion
             scores.sum / scores.size
           end
 
-          def stage_scores(stages) # rubocop:disable Metrics/PerceivedComplexity
+          def stage_scores(stages)
             scores = []
             scores << (stage_passed?(stages[:syntax]) ? 1.0 : 0.0) if stages[:syntax]
             scores << (stage_passed?(stages[:security]) ? 1.0 : 0.0) if stages[:security]
@@ -193,8 +286,12 @@ module Legion
             scores
           end
 
-          def build_result(passed:, verdict:, stages:, issues:, confidence:)
-            { passed: passed, verdict: verdict, confidence: confidence, stages: stages, issues: issues }
+          def build_result(passed:, verdict:, stages:, issues:, confidence:, code: nil, spec_code: nil, extra: {})
+            result = { passed: passed, verdict: verdict, confidence: confidence, stages: stages, issues: issues }
+            result[:code] = code
+            result[:spec_code] = spec_code
+            extra.each { |k, v| result[k] = v unless result.key?(k) }
+            result
           end
         end
       end

data/lib/legion/extensions/eval/runners/evaluation.rb

@@ -5,6 +5,8 @@ module Legion
     module Eval
       module Runners
         module Evaluation
+          extend self # rubocop:disable Style/ModuleFunction
+
           def run_evaluation(evaluator_name:, evaluator_config: {}, inputs: [], **)
             evaluator = build_evaluator(evaluator_name, evaluator_config)
             results = inputs.map.with_index do |row, idx|

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.3.10'
+      VERSION = '0.3.13'
     end
   end
 end

data/lib/legion/extensions/eval.rb

@@ -22,7 +22,7 @@ if defined?(Legion::Transport::Exchange)
   require_relative 'eval/transport/messages/code_review_completed'
 end
 
-require_relative 'eval/actors/code_review_subscriber' if defined?(Legion::Extensions::Actors::Subscription)
+require_relative 'eval/actors/code_review_subscriber'
 
 module Legion
   module Extensions

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.13
 platform: ruby
 authors:
 - Matthew Iverson