legion-llm 0.5.7 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c8ab6d42537aec5471648490e91968d41395206396b69f4cf48d73ac48cf217
-  data.tar.gz: 49ac7e085f62e08b3391e5926e774b920539a1bef03f64b9c7531bc697646ca4
+  metadata.gz: 2ce29d10392a53a933f031e6a19d4437ea57ee6f78710946624275c5c4ac083d
+  data.tar.gz: b372503d1dd95713e3a8b1471715fb10d05aae541e8a9631e6f35fc59b812bb5
 SHA512:
-  metadata.gz: eb1f7f99f9edf09cc108a1d3643c9372ca9e07669dc4f89ff3d2c9d1c5305d1a0e26a8d02316f7edebcb5a9bfe3112a4c8a061f19d2af7bce2ba5649d0e7dccc
-  data.tar.gz: 22983edc5b03faa7692d9c71d5a73ca8d3eccadbf6730b1fd0e7ff9edfb5f2e0b1ea8c926045d3ad0bd6cc4d5fa96900c021964472076b0fce71a5a8fb128be8
+  metadata.gz: 66c5ee8b6f787d69706e32691603b83c895144bc439732810789761680fcb197511a0c0ce31aa210286ef82e279e4d08d44c21b1a7a5cac9c0bd95dfe8573fbf
+  data.tar.gz: ca65b36532cb559c9d0b53ffacc2dfdb404a19e7470f53e933aee6dd686fd97e87e904bd89f10b3fdd593fc78622646c5a0cabe87a1631416af207c2e88dcb50

data/CHANGELOG.md

@@ -1,8 +1,19 @@
 # Legion LLM Changelog
 
-## [Unreleased]
+## [0.5.8] - 2026-03-25
+
+### Added
+- Wire shadow evaluation sampling into `chat_single` dispatch path (closes #3)
+- ToolRegistry spec coverage: 8 examples covering register, dedup, clear, thread safety (closes #4)
+- Arbitrage as router fallback: `Router.resolve` consults `Arbitrage.cheapest_for` when no rules match (closes #5)
+- Batch thread safety: Mutex around queue, priority-sorted flush, auto-flush via `Concurrent::TimerTask` (closes #6)
+- Scheduling deferral in `chat_direct`: defers to Batch during peak hours when scheduling is enabled (closes #7)
+- `publish_escalation_event` now publishes to `Legion::Events` and AMQP transport (closes #8)
+- Arbitrage `quality_floor` filtering via `QualityChecker.model_score` when available (closes #9)
 
 ### Fixed
+- `OffPeak.should_defer?` now checks `Scheduling.enabled?` before returning true (closes #9)
+- Pre-existing ordering-dependent spec failure in `llm_spec.rb` (ToolRegistry bleed)
 - Fix namespace collision: use `::Data.define` instead of `Data.define` in Pipeline Request and Response to prevent resolution to `Legion::Data`
 
 ## [0.5.6] - 2026-03-24

data/lib/legion/llm/arbitrage.rb

@@ -44,7 +44,7 @@ module Legion
           return nil unless enabled?
 
           quality_floor = settings.fetch(:quality_floor, 0.7)
-          eligible = eligible_models(capability: capability, _quality_floor: quality_floor)
+          eligible = eligible_models(capability: capability, quality_floor: quality_floor)
 
           scored = eligible.filter_map do |model|
             cost = estimated_cost(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
@@ -90,17 +90,21 @@ module Legion
         # Returns models eligible for the given capability tier based on quality floor.
         # The quality floor maps capability tiers to minimum acceptable quality scores (0.0-1.0).
         # Models that are local (cost 0) always qualify for :basic capability.
-        def eligible_models(capability:, _quality_floor: 0.7)
+        def eligible_models(capability:, quality_floor: 0.7)
           cap = capability.to_sym
 
-          # Capability tiers determine which models are semantically appropriate.
-          # :reasoning requires frontier models; :basic allows cheap/local models.
-          # _quality_floor reserved for future scoring integration.
           disqualified_for_reasoning = %w[gpt-4o-mini gemini-2.0-flash llama3]
 
-          cost_table.keys.reject do |model|
+          models = cost_table.keys.reject do |model|
             cap == :reasoning && disqualified_for_reasoning.include?(model)
           end
+
+          return models unless defined?(Legion::LLM::QualityChecker) && QualityChecker.respond_to?(:model_score)
+
+          models.select do |model|
+            score = QualityChecker.model_score(model)
+            score.nil? || score >= quality_floor
+          end
         end
       end
     end

data/lib/legion/llm/batch.rb

@@ -5,27 +5,21 @@ require 'securerandom'
 module Legion
   module LLM
     module Batch
+      @mutex = Mutex.new
+      @flush_timer = nil
+
       class << self
-        # Returns true when request batching is enabled in settings.
         def enabled?
           settings.fetch(:enabled, false) == true
         end
 
-        # Enqueues a request for deferred batch processing.
-        #
-        # @param messages [Array<Hash>] chat messages array
-        # @param model    [String]      model to use
-        # @param provider [Symbol, nil] provider override
-        # @param callback [Proc, nil]   called with result hash when batch is flushed
-        # @param priority [Symbol]      :normal or :low (informational only)
-        # @param opts     [Hash]        additional options forwarded to provider
-        # @return [String] batch_request_id
-        def enqueue(messages:, model:, callback: nil, provider: nil, priority: :normal, **opts)
+        def enqueue(messages: nil, model: nil, message: nil, callback: nil, provider: nil, priority: :normal, **opts)
           request_id = SecureRandom.uuid
+          msgs = messages || (message ? [{ role: 'user', content: message }] : [])
 
           entry = {
             id:        request_id,
-            messages:  messages,
+            messages:  msgs,
             model:     model,
             provider:  provider,
             callback:  callback,
@@ -34,30 +28,28 @@ module Legion
             queued_at: Time.now.utc
           }
 
-          queue << entry
-          Legion::Logging.debug "Legion::LLM::Batch enqueued #{request_id} (queue size: #{queue.size})"
+          @mutex.synchronize { queue << entry }
+          ensure_flush_timer
+          Legion::Logging.debug "Legion::LLM::Batch enqueued #{request_id} (queue size: #{queue_size})" if defined?(Legion::Logging)
           request_id
         end
 
-        # Flushes accumulated requests up to max_size.
-        # Groups entries by provider+model and invokes callbacks with a stub result.
-        # In production this would submit to provider batch APIs; here it logs and returns
-        # per-request result hashes for callback delivery.
-        #
-        # @param max_size [Integer] maximum number of requests to flush in one pass
-        # @param max_wait [Integer] only flush entries older than this many seconds (0 = all)
-        # @return [Array<Hash>] array of { id:, status:, result: } hashes
         def flush(max_size: nil, max_wait: nil)
           effective_max  = max_size || settings.fetch(:max_batch_size, 100)
           effective_wait = max_wait || settings.fetch(:window_seconds, 300)
-
           cutoff = Time.now.utc - effective_wait
-          to_flush = queue.select { |e| e[:queued_at] <= cutoff }.first(effective_max)
+
+          to_flush = @mutex.synchronize do
+            ready = queue.select { |e| e[:queued_at] <= cutoff }
+                         .sort_by { |e| priority_rank(e[:priority]) }
+                         .first(effective_max)
+            ready.each { |e| queue.delete(e) }
+            ready
+          end
 
           return [] if to_flush.empty?
 
-          to_flush.each { |e| queue.delete(e) }
-          Legion::Logging.debug "Legion::LLM::Batch flushing #{to_flush.size} request(s)"
+          Legion::Logging.debug "Legion::LLM::Batch flushing #{to_flush.size} request(s)" if defined?(Legion::Logging)
 
           groups = to_flush.group_by { |e| [e[:provider], e[:model]] }
           results = []
@@ -73,14 +65,12 @@ module Legion
           results
         end
 
-        # Returns the current number of requests in the queue.
         def queue_size
-          queue.size
+          @mutex.synchronize { queue.size }
         end
 
-        # Returns a summary of current batch queue state.
         def status
-          entries = queue.dup
+          entries = @mutex.synchronize { queue.dup }
           oldest = entries.min_by { |e| e[:queued_at] }
           {
             enabled:        enabled?,
@@ -92,9 +82,14 @@ module Legion
           }
         end
 
-        # Clears the queue (useful for testing).
         def reset!
-          @queue = []
+          @mutex.synchronize { @queue = [] }
+          stop_flush_timer
+        end
+
+        def stop_flush_timer
+          @flush_timer&.shutdown if @flush_timer.respond_to?(:shutdown)
+          @flush_timer = nil
         end
 
         private
@@ -103,6 +98,30 @@ module Legion
           @queue ||= []
         end
 
+        def priority_rank(priority)
+          case priority.to_sym
+          when :urgent then 0
+          when :normal then 1
+          when :low    then 2
+          else 3
+          end
+        end
+
+        def ensure_flush_timer
+          return if @flush_timer
+          return unless defined?(Concurrent::TimerTask)
+
+          interval = settings.fetch(:window_seconds, 300)
+          return if interval <= 0
+
+          @flush_timer = Concurrent::TimerTask.new(execution_interval: interval) do
+            flush(max_wait: 0)
+          rescue StandardError => e
+            Legion::Logging.warn("Batch auto-flush failed: #{e.message}") if defined?(Legion::Logging)
+          end
+          @flush_timer.execute
+        end
+
         def settings
           llm = Legion::Settings[:llm]
           return {} unless llm.is_a?(Hash)

data/lib/legion/llm/off_peak.rb

@@ -14,6 +14,7 @@ module Legion
 
         def should_defer?(priority: :normal)
           return false if priority.to_sym == :urgent
+          return false unless Scheduling.enabled?
 
           peak_hour?
         end

data/lib/legion/llm/router.rb

@@ -38,7 +38,7 @@ module Legion
             Legion::Logging.debug('Router: no rules matched, resolution is nil')
           end
 
-          resolution
+          resolution || arbitrage_fallback(intent)
         end
 
         def resolve_chain(intent: nil, tier: nil, model: nil, provider: nil, max_escalations: nil)
@@ -80,6 +80,27 @@ module Legion
 
         private
 
+        def arbitrage_fallback(intent)
+          return nil unless defined?(Arbitrage) && Arbitrage.enabled?
+
+          capability = intent&.dig(:capability) || :moderate
+          model = Arbitrage.cheapest_for(capability: capability)
+          return nil unless model
+
+          provider = Arbitrage.cost_table[model] ? infer_provider(model) : nil
+          Legion::Logging.debug("Router: arbitrage fallback selected model=#{model}") if defined?(Legion::Logging)
+          Resolution.new(tier: :cloud, provider: provider || :bedrock, model: model, rule: 'arbitrage_fallback')
+        end
+
+        def infer_provider(model)
+          return :ollama if model.include?('llama')
+          return :bedrock if model.start_with?('us.')
+          return :openai if model.start_with?('gpt')
+          return :google if model.start_with?('gemini')
+
+          :anthropic if model.start_with?('claude')
+        end
+
         def explicit_resolution(tier, provider, model)
           resolved_provider = provider ? provider.to_sym : default_provider_for_tier(tier)
           resolved_model = model || default_model_for_tier(tier)

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.5.7'
+    VERSION = '0.5.8'
   end
 end

data/lib/legion/llm.rb

@@ -121,6 +121,10 @@ module Legion
           end
         end
 
+        urgency = kwargs.delete(:urgency) { :normal }
+        deferred = try_defer(intent: intent, urgency: urgency, model: model, provider: provider, message: message, **kwargs)
+        return deferred if deferred
+
         if defined?(Legion::Logging)
           Legion::Logging.debug "[LLM] chat_direct escalate=#{escalate} message_present=#{!message.nil?} model=#{model} provider=#{provider}"
         end
@@ -321,9 +325,37 @@ module Legion
         Legion::Logging.debug '[LLM] chat_single calling session.ask' if defined?(Legion::Logging)
         response = block ? session.ask(message, &block) : session.ask(message)
         Legion::Logging.debug "[LLM] chat_single response_class=#{response.class} response_nil=#{response.nil?}" if defined?(Legion::Logging)
+
+        if response && !block && ShadowEval.enabled?
+          msgs = session.respond_to?(:messages) ? session.messages : nil
+          maybe_shadow_evaluate(response, msgs, opts[:model])
+        end
+
         response
       end
 
+      def try_defer(intent:, urgency:, model:, provider:, message:, **)
+        return nil unless Scheduling.enabled? && Scheduling.should_defer?(intent: intent || :normal, urgency: urgency)
+        return nil unless defined?(Batch) && Batch.enabled?
+
+        entry_id = Batch.enqueue(model: model, provider: provider, message: message, priority: urgency, **)
+        { deferred: true, batch_id: entry_id, next_off_peak: Scheduling.next_off_peak.iso8601 }
+      end
+
+      def maybe_shadow_evaluate(response, messages, primary_model)
+        return unless ShadowEval.enabled? && ShadowEval.should_sample?
+
+        Thread.new do
+          ShadowEval.evaluate(
+            primary_response: { content: response.respond_to?(:content) ? response.content : response.to_s,
+                                model: primary_model, usage: {} },
+            messages:         messages
+          )
+        rescue StandardError => e
+          Legion::Logging.debug("shadow evaluation failed: #{e.message}") if defined?(Legion::Logging)
+        end
+      end
+
       def chat_with_escalation(model:, provider:, intent:, tier:, max_escalations:, quality_check:, message:, **kwargs)
         chain = Router.resolve_chain(
           intent: intent, tier: tier, model: model, provider: provider,
@@ -390,9 +422,20 @@ module Legion
       end
 
       def publish_escalation_event(history, final_outcome)
-        return unless defined?(Legion::Transport)
+        payload = {
+          outcome:   final_outcome,
+          attempts:  history.size,
+          history:   history,
+          timestamp: Time.now.utc.iso8601
+        }
 
-        Legion::Logging.debug("Escalation event: #{final_outcome}, #{history.size} attempts") if Legion.const_defined?('Logging')
+        Legion::Events.emit('llm.escalation', **payload) if defined?(Legion::Events) && Legion::Events.respond_to?(:emit)
+
+        Legion::Logging.info("Escalation event: #{final_outcome}, #{history.size} attempts") if defined?(Legion::Logging)
+
+        if defined?(Legion::Transport) && Legion::Transport.respond_to?(:connected?) && Legion::Transport.connected?
+          Transport::Messages::EscalationEvent.new(payload).publish
+        end
       rescue StandardError => e
         Legion::Logging.warn("publish_escalation_event failed: #{e.message}") if defined?(Legion::Logging)
         nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.5.7
+  version: 0.5.8
 platform: ruby
 authors:
 - Esity