lex-eval 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6e527b6bd5b88d862de94408ddd8fa8a3ef6fb5a3916b1bb888ab1278c68bf5
-  data.tar.gz: b10274099420ae181d751f6874cc029e76657e08bafa2c2152a3f08fd9896c28
+  metadata.gz: 21965c1d67ff7fe624874e1b2f9ac554cc3ffdbdf9c77158864754a5f5ef6425
+  data.tar.gz: eb2ea1c89bfb257868b63a9c5d8347210b7163d87bb0b9e64786317443f9464f
 SHA512:
-  metadata.gz: 0b8014e5428708f9f04d90fea9cf48b16af90216998aa694c4723ef723dde6365840ceb50a2ef64c65801f613bcf9c3df40c402dd96013f8ce2180e6307cae88
-  data.tar.gz: 973ce2e3f5f9aa4bea347600861ef1e9e4ec741445ca7f4b0ae9142466823920c2eabd1f9608780ad2fa7634fc711b98cfe99825a38ae8de0efdde6616038193
+  metadata.gz: fdfe064f47e393bc4762a74aebc1dd3e079b080df73cc2eb0e2a4446816abf6f14df9b8410049c6cc715ca7027f071f974a51a6f6aa9c688844532b272c8e473
+  data.tar.gz: 7009275960b186e2780c723e54882fe106a0a8c219872f9748c76a70fa94e5602e4cfafecbc3546b19d6ac42d823730f40f0067805339e0af8c2835b504887bf

data/lib/legion/extensions/eval/runners/code_review.rb

@@ -9,10 +9,11 @@ module Legion
       module Runners
         module CodeReview
           extend self
+          include Legion::Logging::Helper
 
           SPEC_TIMEOUT = 30
 
-          def review_generated(code:, spec_code:, context:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          def review_generated(code:, spec_code:, context:) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity, Metrics/MethodLength
             settings = validation_settings
             stages = {}
             issues = []
@@ -48,6 +49,15 @@ module Legion
               issues.concat(stages[:llm_review][:issues] || [])
             end
 
+            # Stage 5: QualityGate (optional, requires lex-factory)
+            qg_settings = settings[:quality_gate] || {}
+            if quality_gate_available? && qg_settings[:enabled] != false
+              stages[:quality_gate] = run_quality_gate(stages, qg_settings)
+              unless stages[:quality_gate][:pass]
+                issues << "quality gate failed: aggregate #{stages[:quality_gate][:aggregate]} below threshold #{stages[:quality_gate][:threshold]}"
+              end
+            end
+
             confidence = calculate_confidence(stages)
             verdict = confidence >= 0.5 ? :approve : :revise
 
@@ -60,7 +70,8 @@ module Legion
             return {} unless defined?(Legion::Settings)
 
             Legion::Settings.dig(:codegen, :self_generate, :validation) || {}
-          rescue StandardError
+          rescue StandardError => e
+            log.warn("validation_settings failed: #{e.message}")
             {}
           end
 
@@ -69,6 +80,7 @@ module Legion
             begin
               RubyVM::InstructionSequence.compile(code)
             rescue SyntaxError => e
+              log.debug("syntax check failed: #{e.message}")
               errors << "code: #{e.message}"
             end
 
@@ -76,6 +88,7 @@ module Legion
               begin
                 RubyVM::InstructionSequence.compile(spec_code)
               rescue SyntaxError => e
+                log.debug("spec syntax check failed: #{e.message}")
                 errors << "spec: #{e.message}"
               end
             end
@@ -104,6 +117,7 @@ module Legion
               { passed: status.success?, output: stdout, errors: stderr, exit_code: status.exitstatus }
             end
           rescue StandardError => e
+            log.warn("spec execution failed: #{e.message}")
             { passed: false, output: '', errors: e.message, exit_code: -1 }
           end
 
@@ -122,6 +136,7 @@ module Legion
               confidence: result[:confidence] || 0.5
             }
           rescue StandardError => e
+            log.warn("llm review failed: #{e.message}")
             { passed: true, issues: ["llm review failed: #{e.message}"], confidence: 0.5 }
           end
 
@@ -129,6 +144,38 @@ module Legion
             defined?(Legion::LLM) && Legion::LLM.respond_to?(:chat)
           end
 
+          def quality_gate_available?
+            defined?(Legion::Extensions::Factory::Helpers::QualityGate)
+          end
+
+          def run_quality_gate(stages, qg_settings)
+            kwargs = quality_gate_dimensions(stages)
+            kwargs[:threshold] = qg_settings[:threshold] if qg_settings[:threshold]
+            Legion::Extensions::Factory::Helpers::QualityGate.score(**kwargs)
+          rescue StandardError => e
+            log.warn("quality gate failed: #{e.message}")
+            { pass: true, aggregate: 1.0, threshold: 0.8, scores: {}, error: e.message }
+          end
+
+          def quality_gate_dimensions(stages)
+            {
+              completeness: stage_passed?(stages[:syntax]) ? 1.0 : 0.0,
+              correctness:  qg_correctness(stages[:specs]),
+              quality:      stages.dig(:llm_review, :confidence) || 1.0,
+              security:     stage_passed?(stages[:security]) ? 1.0 : 0.0
+            }
+          end
+
+          def qg_correctness(specs_stage)
+            return 1.0 unless specs_stage
+
+            stage_passed?(specs_stage) ? 1.0 : 0.3
+          end
+
+          def stage_passed?(stage)
+            stage&.dig(:passed) == true
+          end
+
           def calculate_confidence(stages)
             scores = stage_scores(stages)
             return 0.5 if scores.empty?
@@ -136,12 +183,13 @@ module Legion
             scores.sum / scores.size
           end
 
-          def stage_scores(stages) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+          def stage_scores(stages) # rubocop:disable Metrics/PerceivedComplexity
             scores = []
-            scores << (stages[:syntax]&.dig(:passed) ? 1.0 : 0.0) if stages[:syntax]
-            scores << (stages[:security]&.dig(:passed) ? 1.0 : 0.0) if stages[:security]
-            scores << (stages[:specs]&.dig(:passed) ? 1.0 : 0.3) if stages[:specs]
+            scores << (stage_passed?(stages[:syntax]) ? 1.0 : 0.0) if stages[:syntax]
+            scores << (stage_passed?(stages[:security]) ? 1.0 : 0.0) if stages[:security]
+            scores << (stage_passed?(stages[:specs]) ? 1.0 : 0.3) if stages[:specs]
             scores << (stages.dig(:llm_review, :confidence) || 0.5) if stages[:llm_review]
+            scores << stages.dig(:quality_gate, :aggregate) if stages[:quality_gate]
             scores
           end
 

data/lib/legion/extensions/eval/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Eval
-      VERSION = '0.3.8'
+      VERSION = '0.3.9'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-eval
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - Matthew Iverson