lex-eval 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3214b4f11d574772ad1323a0b37816d45c29ac61e5713f2ee82776ac8906ec09
-  data.tar.gz: b3bd8441ea3a5eda81b9f402e3d7e44e803504002ada86695c41b885f76d624b
+  metadata.gz: 1ba8f0431d907def112bdd8f02a0685002fbfc75754bb658d620ea4889b03c70
+  data.tar.gz: b38fcf01598441f544cf8721d9109010983b357be7085e4e0196edd9872cd16a
 SHA512:
-  metadata.gz: 97601e029b078a0fbf7c1606968e90f5e5f308be477f543e793ff8533eb3ff6ff62716614388eda67416cb671b45e3a07d3d77e58fdf80484944c24bd866cc5f
-  data.tar.gz: c20ab43b7eed22645560f0022094cb8c2b7c9ae0044c452aa57fcdb524e6833bcc30e07b801ec58b52175e2b12871b44e7dace59dfa20499e0a9e6b0529a6ea4
+  metadata.gz: 678d325ad2c47aa74abfe3a8ec4f8c7eb63bb14beec4e8911c7912963ded1cc966dbe57dfe50b721e54cdd85df1ef67e6c04af73004c76f560e9b5b45c0ea132
+  data.tar.gz: abd3cab893f187e0c51929cb9f91fa7f3806d250a339c6c137e6073589d0d6e0a18e9560dcdc9645b39f07c9471e9b93ee5ae0eaf3b74f421501383ba3951078

data/lib/legion/extensions/eval/runners/agentic_review.rb

@@ -23,18 +23,25 @@ module Legion
             required:   %i[confidence recommendation explanation]
           }.freeze
 
-          def review_output(input:, output:, review_prompt: nil, **)
+          def review_output(input:, output:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
             prompt = build_review_message(review_prompt || default_review_prompt, input, output)
-            Legion::LLM.structured(message: prompt, schema: REVIEW_SCHEMA,
-                                   intent: { capability: :reasoning },
-                                   caller: { extension: 'lex-eval', operation: 'agentic_review' })
+            llm_kwargs = {
+              message: prompt, schema: REVIEW_SCHEMA,
+              intent: { capability: :reasoning },
+              caller: { extension: 'lex-eval', operation: 'agentic_review' }
+            }
+            llm_kwargs[:model] = model if model
+            llm_kwargs[:provider] = provider if provider
+            Legion::LLM.structured(**llm_kwargs)
           rescue StandardError => e
+            log.warn(e.message) if respond_to?(:log, true)
             { confidence: 0.0, recommendation: 'reject',
               issues: [], explanation: "review error: #{e.message}" }
           end
 
-          def review_with_escalation(input:, output:, review_prompt: nil, **)
-            review = review_output(input: input, output: output, review_prompt: review_prompt)
+          def review_with_escalation(input:, output:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
+            review = review_output(input: input, output: output, review_prompt: review_prompt,
+                                   model: model, provider: provider)
             action, priority = determine_escalation(review[:confidence])
 
             return review.merge(action: :auto_approve, escalated: false) if action == :auto_approve
@@ -42,9 +49,11 @@ module Legion
             review.merge(action: action, escalated: true, priority: priority)
           end
 
-          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, **)
-            review_a = review_output(input: input, output: output_a, review_prompt: review_prompt)
-            review_b = review_output(input: input, output: output_b, review_prompt: review_prompt)
+          def review_experiment(input:, output_a:, output_b:, review_prompt: nil, model: nil, provider: nil, **) # rubocop:disable Metrics/ParameterLists
+            review_a = review_output(input: input, output: output_a, review_prompt: review_prompt,
+                                     model: model, provider: provider)
+            review_b = review_output(input: input, output: output_b, review_prompt: review_prompt,
+                                     model: model, provider: provider)
 
             conf_a = review_a[:confidence] || 0.0
             conf_b = review_b[:confidence] || 0.0
@@ -64,6 +73,7 @@ module Legion
               review_a: review_a,
               review_b: review_b }
           rescue StandardError => e
+            log.warn(e.message) if respond_to?(:log, true)
             { reviewed: false, reason: "experiment error: #{e.message}" }
           end
 

data/lib/legion/extensions/eval/runners/code_review.rb

@@ -13,7 +13,7 @@ module Legion
 
           SPEC_TIMEOUT = 30
 
-          def review_generated(code:, spec_code:, context:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
+          def review_generated(code:, spec_code:, context:, review_k: nil, review_models: nil) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
             settings = validation_settings
             stages = {}
             issues = []
@@ -45,7 +45,13 @@ module Legion
 
             # Stage 4: LLM review (optional)
             if settings[:llm_review] && llm_available?
-              stages[:llm_review] = llm_review(code, context)
+              k = review_k || default_review_k
+              models = review_models || default_review_models
+              stages[:llm_review] = if k > 1
+                                      adversarial_llm_review(code, context, count: k, models: models)
+                                    else
+                                      llm_review(code, context, model_spec: build_model_assignments(1, models)&.first)
+                                    end
               issues.concat(stages[:llm_review][:issues] || [])
             end
 
@@ -75,6 +81,80 @@ module Legion
             {}
           end
 
+          def default_review_k
+            return 1 unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:codegen, :self_generate, :validation, :review_k) || 1
+          rescue StandardError => e
+            log.warn(e.message)
+            1
+          end
+
+          def default_review_models
+            return [] unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:codegen, :self_generate, :validation, :review_models) || []
+          rescue StandardError => e
+            log.warn(e.message)
+            []
+          end
+
+          def provider_available?(provider_sym)
+            return false unless defined?(Legion::Settings)
+
+            Legion::Settings.dig(:llm, :providers, provider_sym, :enabled) == true
+          rescue StandardError => e
+            log.warn(e.message)
+            false
+          end
+
+          def build_model_assignments(count, models) # rubocop:disable Metrics/PerceivedComplexity
+            return Array.new(count) { nil } if models.nil? || models.empty?
+
+            available = models.select do |spec|
+              next false unless spec.is_a?(Hash)
+
+              provider_sym = spec[:provider]&.to_sym
+              if provider_sym && !provider_available?(provider_sym)
+                log.warn("review provider #{provider_sym} not available, skipping")
+                false
+              else
+                true
+              end
+            end
+
+            return Array.new(count) { nil } if available.empty?
+
+            Array.new(count) { |i| available[i % available.size] }
+          end
+
+          def adversarial_llm_review(code, context, count:, models: []) # rubocop:disable Metrics/PerceivedComplexity
+            assignments = build_model_assignments(count, models)
+
+            reviews = assignments.map { |spec| llm_review(code, context, model_spec: spec) }
+
+            approvals = reviews.count { |r| r[:confidence] >= 0.5 }
+            rejections = count - approvals
+            all_issues = reviews.flat_map { |r| r[:issues] || [] }.uniq
+
+            avg_confidence = reviews.sum { |r| r[:confidence] || 0.0 } / reviews.size
+
+            {
+              passed:     approvals > rejections,
+              issues:     all_issues,
+              confidence: avg_confidence,
+              k:          count,
+              approvals:  approvals,
+              rejections: rejections,
+              reviews:    reviews
+            }
+          rescue StandardError => e
+            log.warn("adversarial review failed: #{e.message}")
+            fallback = llm_review(code, context)
+            fallback.merge(k: count, approvals: (fallback[:passed] ? 1 : 0),
+                           rejections: (fallback[:passed] ? 0 : 1), reviews: [fallback])
+          end
+
           def check_syntax(code, spec_code)
             errors = []
             begin
@@ -121,19 +201,28 @@ module Legion
             { passed: false, output: '', errors: e.message, exit_code: -1 }
           end
 
-          def llm_review(code, context)
+          def llm_review(code, context, model_spec: nil) # rubocop:disable Metrics/PerceivedComplexity
             return { passed: true, issues: [], confidence: 0.5 } unless defined?(Runners::AgenticReview)
 
+            extra_kwargs = {}
+            if model_spec
+              extra_kwargs[:model] = model_spec[:model] if model_spec[:model]
+              extra_kwargs[:provider] = model_spec[:provider] if model_spec[:provider]
+            end
+
             result = Runners::AgenticReview.review_output(
               input:         context,
               output:        code,
-              review_prompt: 'Review this generated Ruby code for correctness, safety, and Legion conventions.'
+              review_prompt: 'Review this generated Ruby code for correctness, safety, and Legion conventions.',
+              **extra_kwargs
             )
 
             {
-              passed:     result[:reviewed] != false,
+              passed:     result[:recommendation] == 'approve',
               issues:     result[:issues] || [],
-              confidence: result[:confidence] || 0.5
+              confidence: result[:confidence] || 0.5,
+              provider:   model_spec&.dig(:provider),
+              model:      model_spec&.dig(:model)
             }
           rescue StandardError => e
             log.warn("llm review failed: #{e.message}")

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.3.10'
+      VERSION = '0.3.11'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Matthew Iverson