legion-llm 0.3.12 → 0.3.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0551af82013a885240cd8d38ba1f991d470110d308925a8f4848b8650376d252
-  data.tar.gz: fc2da425ddafa426f89375dbffd9afccc2c5d318207ed2bff0513dc57cf7dc07
+  metadata.gz: f6dc45bc6e985a3a6399ba3ed860bfb1ac9d3d9a0f31dda55a2f812d3c46e7cb
+  data.tar.gz: c3db21154b0b43de08e3e23b24416d9a7dc26a58eb10beb19835845b6ad83500
 SHA512:
-  metadata.gz: 3ebfd45a16cd899050c44c0e53b0ae9952c8c87f46381b0af4356cda6d03ebff05084c3d66923ffd9f3012674b41005e080fa0350cdb4a2f799cbaa83e1cd4dc
-  data.tar.gz: bfb977400e5c78caa90012af604ec47c7b4be94e1d268cfceb3b240b1e9731f678b4a0f6dc30de4df6e014e4714701b15f554e9a5858b3a94754aedaaa67da84
+  metadata.gz: 6bd0700aee69aab3d7dad4e3266855d6ddf28de1574a9b1e48e972b653f4af509720e53b2d8c34e84ac9599a325b539c5fc6c7ac765e6c62a846a40e2b6b9519
+  data.tar.gz: c2ffe0842728637165668508a68a690eb0a00596710108b4685f47e4fa8b78f24e634ec652e11d7f86ace856f0166299c6827e7bb7a4f1e9ed6e491ed97ca559

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Legion LLM Changelog
 
+## [0.3.13] - 2026-03-21
+
+### Added
+- `Legion::LLM::Hooks::RagGuard` module with `check_rag_faithfulness` for post-generation RAG faithfulness evaluation via lex-eval
+- `Legion::LLM::Hooks::ResponseGuard` module with `guard_response` as the central dispatch point for post-generation safety checks
+- Response guard wired into `_dispatch_chat`: fires when `Legion::Settings[:llm][:response_guards][:enabled]` is true, attaches `_guard_result` metadata to the response hash without blocking
+- RAG guard skips gracefully when lex-eval is unavailable (returns `reason: :eval_unavailable`) or context is not provided (returns `reason: :no_context`)
+- Settings keys: `llm.rag_guard.enabled`, `llm.rag_guard.threshold` (default 0.7), `llm.rag_guard.evaluators` (default `[:faithfulness, :rag_relevancy]`)
+- 19 new specs in `spec/legion/llm/hooks/rag_guard_spec.rb` and `spec/legion/llm/hooks/response_guard_spec.rb`
+
 ## [0.3.12] - 2026-03-19
 
 ### Added

data/lib/legion/llm/hooks/rag_guard.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Hooks
+      module RagGuard
+        class << self
+          def check_rag_faithfulness(response:, context:, threshold: nil, evaluators: nil, **)
+            return { faithful: true, reason: :eval_unavailable } unless eval_available?
+
+            resolved_threshold = threshold || settings_threshold
+            resolved_evaluators = evaluators || settings_evaluators
+
+            scores = {}
+            flagged = []
+
+            resolved_evaluators.each do |evaluator_name|
+              score = run_evaluator(evaluator_name, response: response, context: context)
+              scores[evaluator_name] = score
+              flagged << evaluator_name if score < resolved_threshold
+            end
+
+            faithful = flagged.empty?
+            details = build_details(scores, resolved_threshold, faithful)
+
+            { faithful: faithful, scores: scores, flagged_evaluators: flagged, details: details }
+          rescue StandardError => e
+            Legion::Logging.warn "RagGuard evaluation error: #{e.message}" if logging_available?
+            { faithful: true, reason: :eval_error }
+          end
+
+          private
+
+          def eval_available?
+            defined?(Legion::Extensions::Eval::Client)
+          end
+
+          def logging_available?
+            Legion.const_defined?('Logging')
+          end
+
+          def settings_threshold
+            val = Legion::Settings.dig(:llm, :rag_guard, :threshold) if Legion.const_defined?('Settings')
+            val || 0.7
+          end
+
+          def settings_evaluators
+            val = Legion::Settings.dig(:llm, :rag_guard, :evaluators) if Legion.const_defined?('Settings')
+            val || %i[faithfulness rag_relevancy]
+          end
+
+          def run_evaluator(evaluator_name, response:, context:)
+            client = Legion::Extensions::Eval::Client.new
+            result = client.run_evaluation(
+              evaluator_name: evaluator_name,
+              inputs:         [{ input: context.to_s, output: response.to_s, expected: nil }]
+            )
+            result.dig(:summary, :avg_score) || 0.0
+          rescue StandardError
+            0.0
+          end
+
+          def build_details(scores, threshold, faithful)
+            score_parts = scores.map { |k, v| "#{k}=#{v.round(3)}" }.join(', ')
+            status = faithful ? 'passed' : 'failed'
+            "RAG faithfulness check #{status} (threshold=#{threshold}): #{score_parts}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/hooks/response_guard.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Hooks
+      module ResponseGuard
+        GUARD_REGISTRY = {
+          rag: RagGuard
+        }.freeze
+
+        class << self
+          def guard_response(response:, context: nil, guards: [:rag], **)
+            guard_results = {}
+
+            guards.each do |guard_name|
+              guard_mod = GUARD_REGISTRY[guard_name.to_sym]
+              next unless guard_mod
+
+              guard_results[guard_name] = dispatch_guard(guard_mod, guard_name,
+                                                         response: response, context: context)
+            end
+
+            passed = guard_results.values.all? { |r| r[:faithful] != false }
+
+            { passed: passed, guards: guard_results }
+          rescue StandardError => e
+            Legion::Logging.warn "ResponseGuard error: #{e.message}" if Legion.const_defined?('Logging')
+            { passed: true, guards: {} }
+          end
+
+          private
+
+          def dispatch_guard(guard_mod, guard_name, response:, context:)
+            case guard_name.to_sym
+            when :rag
+              return { faithful: true, reason: :no_context } if context.nil?
+
+              guard_mod.check_rag_faithfulness(response: response, context: context)
+            else
+              guard_mod.check(response: response, context: context)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/hooks.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'legion/llm/hooks/rag_guard'
+require 'legion/llm/hooks/response_guard'
+
 module Legion
   module LLM
     module Hooks

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.3.12'
+    VERSION = '0.3.13'
   end
 end

data/lib/legion/llm.rb

@@ -184,7 +184,7 @@ module Legion
 
       private
 
-      def _dispatch_chat(model:, provider:, intent:, tier:, escalate:, max_escalations:, quality_check:, message:, **)
+      def _dispatch_chat(model:, provider:, intent:, tier:, escalate:, max_escalations:, quality_check:, message:, **kwargs)
         messages = message.is_a?(Array) ? message : [{ role: 'user', content: message.to_s }]
         resolved_model = model || settings[:default_model]
 
@@ -196,11 +196,11 @@ module Legion
         result = if gateway_loaded? && message
                    gateway_chat(model: model, provider: provider, intent: intent,
                                 tier: tier, message: message, escalate: escalate,
-                                max_escalations: max_escalations, quality_check: quality_check, **)
+                                max_escalations: max_escalations, quality_check: quality_check, **kwargs)
                  else
                    chat_direct(model: model, provider: provider, intent: intent, tier: tier,
                                escalate: escalate, max_escalations: max_escalations,
-                               quality_check: quality_check, message: message, **)
+                               quality_check: quality_check, message: message, **kwargs)
                  end
 
         if defined?(Legion::LLM::Hooks)
@@ -208,6 +208,8 @@ module Legion
           return blocked[:response] if blocked
         end
 
+        result = apply_response_guards(result, kwargs) if response_guards_enabled? && result.is_a?(Hash)
+
         result
       end
 
@@ -370,6 +372,24 @@ module Legion
         nil
       end
 
+      def response_guards_enabled?
+        settings.dig(:response_guards, :enabled) == true
+      end
+
+      def apply_response_guards(result, kwargs)
+        context = kwargs[:context]
+        response_text = result[:response] || result[:content]
+        guard_result = Hooks::ResponseGuard.guard_response(
+          response: response_text, context: context
+        )
+
+        Legion::Logging.warn "Response guard failed: #{guard_result.inspect}" if !guard_result[:passed] && Legion.const_defined?('Logging')
+
+        result.merge(_guard_result: guard_result)
+      rescue StandardError
+        result
+      end
+
       def cacheable?(cache_opt, temperature, message)
         cache_opt != false && temperature.to_f.zero? && message && Cache.enabled?
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.13
 platform: ruby
 authors:
 - Esity
@@ -141,6 +141,8 @@ files:
 - lib/legion/llm/escalation_history.rb
 - lib/legion/llm/helpers/llm.rb
 - lib/legion/llm/hooks.rb
+- lib/legion/llm/hooks/rag_guard.rb
+- lib/legion/llm/hooks/response_guard.rb
 - lib/legion/llm/providers.rb
 - lib/legion/llm/quality_checker.rb
 - lib/legion/llm/response_cache.rb