legion-llm 0.5.16 → 0.5.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dea674b5405be2c2863f1c6dd568f21ec8baad8db42eeaa457cd6dcdc881bc8
-  data.tar.gz: ee16678e6be6bc612d906bdd754d7e3db79f803c52b465fe3fb2ed762812aa20
+  metadata.gz: '0915f8beeff34fe070509f7d5bb3fe78a242213c7fc6b406905d48a264527fc1'
+  data.tar.gz: 01c1189bac8f90310518c1650b5621fd75b1a8cbf503c2b02277fef0c195f986
 SHA512:
-  metadata.gz: 0a743021a3a3540290cfc4ea3c119fdc42bbba38eb5115b2883fefc6a4da0bceda04c45136c72d233559d9de53c41af198bfa057eed5579fc345121fade8cd74
-  data.tar.gz: 58ba674f0aa898bd75895bfaeb93b5f31bf2aa7f6dc0dbbc8d3f0afcb96cbfcabb80b68798c2c8758f8df8726c9bfd86a567e1e531728f45f91af96b53e15e7b
+  metadata.gz: aadc8ac33d46d40e470be524ca9f59f252c021425ab56ea1f19072e9fdd95bc2f96b74096193fbe5b0572417aeca2daf026b9d3e6486bd59f527d0ee98a108d3
+  data.tar.gz: f55c014a191d403b991ce8280e5d50b689adfaf7a5bb6f320add35555dbbefe615b860a2754507a22b830b413cd52ad819bc181448ddc0f36f6c970ab60b86c2

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Legion LLM Changelog
 
+## [0.5.17] - 2026-03-28
+
+### Added
+- `Legion::LLM::ConfidenceScore` value object (`lib/legion/llm/confidence_score.rb`): immutable struct with `score` (Float 0.0–1.0), `band` (`:very_low/:low/:medium/:high/:very_high`), `source` (`:heuristic/:logprobs/:caller_provided`), and `signals` hash. `#at_least?(band)` for band comparison. `BAND_ORDER` constant for ordered band comparison.
+- `Legion::LLM::ConfidenceScorer` module (`lib/legion/llm/confidence_scorer.rb`): computes `ConfidenceScore` from three strategy sources in priority order — (1) caller-provided score via `confidence_score:` option, (2) model-native logprobs (detected via `class.method_defined?(:logprobs)` to avoid test-double interference), (3) heuristic analysis (refusal, truncation, repetition, too_short, json_parse_failure, hedging language penalties; structured output bonus for valid JSON). Band boundaries are read from `Legion::Settings[:llm][:confidence][:bands]` at call time, per-call overrides accepted via `confidence_bands:` option.
+- `Legion::LLM::Pipeline::Steps::ConfidenceScoring` module (`lib/legion/llm/pipeline/steps/confidence_scoring.rb`): new pipeline step `step_confidence_scoring` inserted after `response_normalization`. Reads `confidence_score:`, `confidence_bands:`, and `quality_threshold:` from `request.extra`; propagates `json_expected:` from `request.response_format`. Errors are soft-caught (appended to `@warnings`, step skipped).
+- `confidence_defaults` settings method: band boundaries `{ low: 0.3, medium: 0.5, high: 0.7, very_high: 0.9 }` under `Legion::Settings[:llm][:confidence][:bands]`.
+- `confidence_score` attr_reader on `Pipeline::Executor` for post-pipeline inspection.
+- `quality:` field of `Pipeline::Response` is now populated with `@confidence_score.to_h` (score, band, source, signals).
+- 54 new specs across `confidence_score_spec.rb`, `confidence_scorer_spec.rb`, `confidence_settings_spec.rb`, and `pipeline/steps/confidence_scoring_spec.rb`.
+
+### Changed
+- `Pipeline::Executor::STEPS` and `POST_PROVIDER_STEPS` now include `:confidence_scoring` after `:response_normalization`.
+- `Legion::LLM.start` now requires `confidence_score` and `confidence_scorer` after `quality_checker`.
+
 ## [0.5.16] - 2026-03-28
 
 ### Fixed

data/lib/legion/llm/confidence_score.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    # Immutable value object representing a scored confidence level for an LLM response.
+    #
+    # score - Float in [0.0, 1.0]
+    # band  - Symbol: :very_low, :low, :medium, :high, :very_high
+    # source - Symbol: :heuristic, :logprobs, :caller_provided
+    # signals - Hash of contributing signals and their raw values (informational)
+    ConfidenceScore = ::Data.define(:score, :band, :source, :signals) do
+      def self.build(score:, bands:, source: :heuristic, signals: {})
+        clamped = score.to_f.clamp(0.0, 1.0)
+        new(
+          score:   clamped,
+          band:    classify(clamped, bands),
+          source:  source,
+          signals: signals
+        )
+      end
+
+      # Returns true when the band is at or above the given band name.
+      def at_least?(band_name)
+        Legion::LLM::ConfidenceScore::BAND_ORDER.index(band) >= Legion::LLM::ConfidenceScore::BAND_ORDER.index(band_name.to_sym)
+      end
+
+      def to_h
+        { score: score, band: band, source: source, signals: signals }
+      end
+
+      class << self
+        private
+
+        def classify(score, bands)
+          return :very_low  if score < bands.fetch(:low,       0.3)
+          return :low       if score < bands.fetch(:medium,    0.5)
+          return :medium    if score < bands.fetch(:high,      0.7)
+          return :high      if score < bands.fetch(:very_high, 0.9)
+
+          :very_high
+        end
+      end
+    end
+
+    # Band ordering from lowest to highest — defined outside the ::Data.define block
+    # so it is accessible as Legion::LLM::ConfidenceScore::BAND_ORDER.
+    ConfidenceScore::BAND_ORDER = %i[very_low low medium high very_high].freeze
+  end
+end

data/lib/legion/llm/confidence_scorer.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    # Computes a ConfidenceScore for an LLM response using available signals.
+    #
+    # Strategy selection (in priority order):
+    #   1. logprobs  — native model confidence from token log-probabilities (when available)
+    #   2. caller    — caller-provided score passed via options[:confidence_score]
+    #   3. heuristic — derived from response content characteristics
+    #
+    # Band boundaries are read from Legion::Settings[:llm][:confidence][:bands] when
+    # Legion::Settings is available, otherwise the DEFAULT_BANDS constants are used.
+    # Per-call overrides can be passed as options[:confidence_bands].
+    module ConfidenceScorer
+      # Default band boundaries. Keys are the *lower* boundary of that band name:
+      #   score <  :low       -> :very_low
+      #   score <  :medium    -> :low
+      #   score <  :high      -> :medium
+      #   score <  :very_high -> :high
+      #   score >= :very_high -> :very_high
+      DEFAULT_BANDS = {
+        low:       0.3,
+        medium:    0.5,
+        high:      0.7,
+        very_high: 0.9
+      }.freeze
+
+      # Penalty weights used in heuristic scoring.
+      HEURISTIC_WEIGHTS = {
+        refusal:            -0.8,
+        empty:              -1.0,
+        truncated:          -0.4,
+        repetition:         -0.5,
+        json_parse_failure: -0.6,
+        too_short:          -0.3
+      }.freeze
+
+      # Bonus applied when structured output parse succeeds.
+      STRUCTURED_OUTPUT_BONUS = 0.1
+
+      # Hedging language patterns that reduce confidence.
+      HEDGING_PATTERNS = [
+        /\b(?:I think|I believe|I'm not sure|I'm uncertain|it seems|it appears|maybe|perhaps|possibly|probably|I guess|I assume)\b/i,
+        /\bnot (?:certain|sure|definite|confirmed)\b/i,
+        /\bunclear\b/i,
+        /\bcould be\b/i
+      ].freeze
+
+      class << self
+        # Compute a ConfidenceScore for the given raw_response.
+        #
+        # raw_response - the RubyLLM response object (must respond to #content)
+        # options      - Hash:
+        #   :confidence_score  - Float  caller-provided score (bypasses heuristics)
+        #   :confidence_bands  - Hash   per-call band overrides
+        #   :json_expected     - Boolean whether JSON output was expected
+        #   :quality_result    - QualityResult from QualityChecker (optional, avoids re-running checks)
+        #
+        # Returns a ConfidenceScore.
+        def score(raw_response, **options)
+          bands = resolve_bands(options[:confidence_bands])
+
+          if (caller_score = options[:confidence_score])
+            return ConfidenceScore.build(
+              score:   caller_score.to_f,
+              bands:   bands,
+              source:  :caller_provided,
+              signals: { caller_provided: caller_score.to_f }
+            )
+          end
+
+          if (lp = extract_logprobs(raw_response))
+            return ConfidenceScore.build(
+              score:   lp,
+              bands:   bands,
+              source:  :logprobs,
+              signals: { avg_logprob: lp }
+            )
+          end
+
+          heuristic_score(raw_response, bands: bands, options: options)
+        end
+
+        private
+
+        # Resolve band configuration.  Per-call overrides win, then settings,
+        # then DEFAULT_BANDS.
+        def resolve_bands(per_call_override)
+          base = settings_bands
+          return base.merge(per_call_override) if per_call_override.is_a?(Hash)
+
+          base
+        end
+
+        def settings_bands
+          return DEFAULT_BANDS unless defined?(Legion::Settings)
+
+          raw = Legion::Settings[:llm]
+          return DEFAULT_BANDS unless raw.is_a?(Hash)
+
+          conf = raw.dig(:confidence, :bands)
+          return DEFAULT_BANDS unless conf.is_a?(Hash)
+
+          DEFAULT_BANDS.merge(conf.transform_keys(&:to_sym))
+        end
+
+        # Attempt to derive a score from logprobs attached to the response.
+        # RubyLLM does not currently expose logprobs in its standard interface,
+        # but some providers return them in extra metadata.  We probe the response
+        # object defensively to avoid unexpected-message errors from test doubles.
+        def extract_logprobs(raw_response)
+          lp = probe_logprobs(raw_response)
+          return nil unless lp.is_a?(Array) && !lp.empty?
+
+          # lp is expected to be an array of token log-probability floats (negative values).
+          avg_lp = lp.sum.to_f / lp.size
+          # Convert average log-probability to a probability-like score in [0, 1].
+          # avg_lp is in (-inf, 0]; e^0 = 1.0 (perfect), e^(-5) ≈ 0.007 (very uncertain).
+          # We clamp at -5 so very negative values still map to > 0.
+          Math.exp([avg_lp, -5.0].max)
+        rescue StandardError
+          nil
+        end
+
+        # Safely probe a response object for logprobs.
+        # Checks method_defined? on the concrete class first (not via stubs or method_missing)
+        # to avoid triggering MockExpectationError on RSpec test doubles.
+        def probe_logprobs(raw_response)
+          klass = raw_response.class
+          lp = raw_response.logprobs if klass.method_defined?(:logprobs)
+          lp ||= raw_response.metadata&.dig(:logprobs) if klass.method_defined?(:metadata)
+          lp
+        rescue StandardError
+          nil
+        end
+
+        def heuristic_score(raw_response, bands:, options:)
+          signals  = {}
+          penalty  = 0.0
+          content  = raw_response.respond_to?(:content) ? raw_response.content.to_s : ''
+
+          # Use pre-computed QualityResult when available to avoid duplicate work.
+          quality_result = options[:quality_result]
+
+          if content.strip.empty?
+            signals[:empty] = true
+            penalty += HEURISTIC_WEIGHTS[:empty].abs
+          else
+            failures = quality_result ? quality_result.failures : detect_failures(content, options)
+
+            failures.each do |failure|
+              weight = HEURISTIC_WEIGHTS[failure]
+              next unless weight
+
+              signals[failure] = true
+              penalty += weight.abs
+            end
+
+            hedges = count_hedges(content)
+            if hedges.positive?
+              hedge_penalty = [hedges * 0.05, 0.3].min
+              signals[:hedging] = hedges
+              penalty += hedge_penalty
+            end
+
+            if options[:json_expected] && !failures.include?(:json_parse_failure)
+              signals[:structured_output_valid] = true
+              penalty -= STRUCTURED_OUTPUT_BONUS
+            end
+          end
+
+          raw_score = [1.0 - penalty.clamp(0.0, 1.0), 0.0].max
+          ConfidenceScore.build(score: raw_score, bands: bands, source: :heuristic, signals: signals)
+        end
+
+        def detect_failures(content, options)
+          return [] if content.strip.empty?
+
+          failures = []
+          threshold = options.fetch(:quality_threshold, QualityChecker::DEFAULT_QUALITY_THRESHOLD)
+          failures << :too_short if content.length < threshold
+          failures << :truncated if truncated?(content)
+          failures << :refusal   if refusal?(content)
+          failures << :repetition if repetitive?(content)
+          failures << :json_parse_failure if options[:json_expected] && !valid_json?(content)
+          failures
+        end
+
+        def truncated?(content)
+          return false if content.length < 100
+
+          last_chars = content[-3..]
+          last_chars&.match?(/\w{3}\z/) &&
+            !content.end_with?('.', '!', '?', '`', '"', "'", ')', ']', '}', "\n")
+        end
+
+        def refusal?(content)
+          first_line = content.lines.first.to_s
+          QualityChecker::REFUSAL_PATTERNS.any? { |pat| first_line.match?(pat) }
+        end
+
+        def repetitive?(content)
+          return false if content.length < QualityChecker::REPETITION_MIN_LENGTH * QualityChecker::REPETITION_THRESHOLD
+
+          seen = {}
+          step = QualityChecker::REPETITION_MIN_LENGTH
+          (0..(content.length - step)).step(step) do |i|
+            chunk = content[i, step]
+            seen[chunk] = (seen[chunk] || 0) + 1
+            return true if seen[chunk] >= QualityChecker::REPETITION_THRESHOLD
+          end
+          false
+        end
+
+        def valid_json?(content)
+          ::JSON.parse(content)
+          true
+        rescue ::JSON::ParserError
+          false
+        end
+
+        def count_hedges(content)
+          HEDGING_PATTERNS.sum { |pat| content.scan(pat).size }
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/pipeline/executor.rb

@@ -12,17 +12,18 @@ module Legion
         include Steps::RagContext
 
         attr_reader :request, :profile, :timeline, :tracing, :enrichments,
-                    :audit, :warnings, :discovered_tools
+                    :audit, :warnings, :discovered_tools, :confidence_score
 
         include Steps::McpDiscovery
         include Steps::ToolCalls
         include Steps::KnowledgeCapture
+        include Steps::ConfidenceScoring
 
         STEPS = %i[
           tracing_init idempotency conversation_uuid context_load
           rbac classification billing gaia_advisory rag_context mcp_discovery
           routing request_normalization provider_call response_normalization
-          tool_calls context_store post_response knowledge_capture response_return
+          confidence_scoring tool_calls context_store post_response knowledge_capture response_return
         ].freeze
 
         PRE_PROVIDER_STEPS = %i[
@@ -32,7 +33,7 @@ module Legion
         ].freeze
 
         POST_PROVIDER_STEPS = %i[
-          response_normalization tool_calls context_store post_response knowledge_capture response_return
+          response_normalization confidence_scoring tool_calls context_store post_response knowledge_capture response_return
         ].freeze
 
         def initialize(request)
@@ -46,9 +47,10 @@ module Legion
           @timestamps   = { received: Time.now }
           @raw_response = nil
           @exchange_id  = nil
-          @discovered_tools = []
+          @discovered_tools  = []
           @resolved_provider = nil
           @resolved_model    = nil
+          @confidence_score  = nil
         end
 
         def call
@@ -310,7 +312,8 @@ module Legion
             caller:          @request.caller,
             classification:  @request.classification,
             billing:         @request.billing,
-            test:            @request.test
+            test:            @request.test,
+            quality:         @confidence_score&.to_h
           )
         end
       end

data/lib/legion/llm/pipeline/steps/confidence_scoring.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Pipeline
+      module Steps
+        module ConfidenceScoring
+          def step_confidence_scoring
+            return unless @raw_response
+
+            opts = {
+              json_expected:     @request.response_format&.dig(:type) == :json,
+              quality_threshold: @request.extra&.dig(:quality_threshold),
+              confidence_score:  @request.extra&.dig(:confidence_score),
+              confidence_bands:  @request.extra&.dig(:confidence_bands)
+            }.compact
+
+            @confidence_score = ConfidenceScorer.score(@raw_response, **opts)
+
+            @timeline.record(
+              category: :internal, key: 'confidence:scored',
+              direction: :internal,
+              detail: "score=#{@confidence_score.score.round(3)} band=#{@confidence_score.band} source=#{@confidence_score.source}",
+              from: 'pipeline', to: 'pipeline'
+            )
+          rescue StandardError => e
+            @warnings << "confidence_scoring error: #{e.message}"
+            @confidence_score = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/pipeline/steps.rb

@@ -20,3 +20,4 @@ require_relative 'steps/tool_calls'
 require_relative 'steps/rag_context'
 require_relative 'steps/rag_guard'
 require_relative 'steps/knowledge_capture'
+require_relative 'steps/confidence_scoring'

data/lib/legion/llm/settings.rb

@@ -13,6 +13,7 @@ module Legion
           default_provider: nil,
           providers:        providers,
           routing:          routing_defaults,
+          confidence:       confidence_defaults,
           discovery:        discovery_defaults,
           gateway:          gateway_defaults,
           daemon:           daemon_defaults,
@@ -25,6 +26,17 @@ module Legion
         }
       end
 
+      def self.confidence_defaults
+        {
+          bands: {
+            low:       0.3,
+            medium:    0.5,
+            high:      0.7,
+            very_high: 0.9
+          }
+        }
+      end
+
       def self.daemon_defaults
         {
           url:     nil,

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.5.16'
+    VERSION = '0.5.17'
   end
 end

data/lib/legion/llm.rb

@@ -9,6 +9,8 @@ require 'legion/llm/providers'
 require 'legion/llm/router'
 require 'legion/llm/compressor'
 require 'legion/llm/quality_checker'
+require 'legion/llm/confidence_score'
+require 'legion/llm/confidence_scorer'
 require 'legion/llm/escalation_history'
 require 'legion/llm/hooks'
 require 'legion/llm/cache'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.5.16
+  version: 0.5.17
 platform: ruby
 authors:
 - Esity
@@ -214,6 +214,8 @@ files:
 - lib/legion/llm/cache.rb
 - lib/legion/llm/claude_config_loader.rb
 - lib/legion/llm/compressor.rb
+- lib/legion/llm/confidence_score.rb
+- lib/legion/llm/confidence_scorer.rb
 - lib/legion/llm/conversation_store.rb
 - lib/legion/llm/cost_estimator.rb
 - lib/legion/llm/cost_tracker.rb
@@ -249,6 +251,7 @@ files:
 - lib/legion/llm/pipeline/steps.rb
 - lib/legion/llm/pipeline/steps/billing.rb
 - lib/legion/llm/pipeline/steps/classification.rb
+- lib/legion/llm/pipeline/steps/confidence_scoring.rb
 - lib/legion/llm/pipeline/steps/gaia_advisory.rb
 - lib/legion/llm/pipeline/steps/knowledge_capture.rb
 - lib/legion/llm/pipeline/steps/mcp_discovery.rb