legion-llm 0.3.13 → 0.3.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6dc45bc6e985a3a6399ba3ed860bfb1ac9d3d9a0f31dda55a2f812d3c46e7cb
-  data.tar.gz: c3db21154b0b43de08e3e23b24416d9a7dc26a58eb10beb19835845b6ad83500
+  metadata.gz: 16ae90179fe84f5fdef3459c5463517048a71481e53c778c6be53ef8a0e4f078
+  data.tar.gz: 6bc1cdebbf9807443e057748abd11e6fb41694a68666bbc1da2dee3fa4ead10a
 SHA512:
-  metadata.gz: 6bd0700aee69aab3d7dad4e3266855d6ddf28de1574a9b1e48e972b653f4af509720e53b2d8c34e84ac9599a325b539c5fc6c7ac765e6c62a846a40e2b6b9519
-  data.tar.gz: c2ffe0842728637165668508a68a690eb0a00596710108b4685f47e4fa8b78f24e634ec652e11d7f86ace856f0166299c6827e7bb7a4f1e9ed6e491ed97ca559
+  metadata.gz: 1a5a14010e8b18f19f38d94a64ebfbc5a1d0f0ad589aa6c87d2155ee2705f1369c68e56f3697075bab2a4a43a2fc67cbf3797845f32a8214e156ca2003f64ccb
+  data.tar.gz: 6251af13334ead29cb2d53cd76137d3627d28f23fce0787816b037718611f1f9ca26bc4b48ea49c33ca0747e9a1c875c2641ae577f4e557b65f68b9d1adfe32b

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Legion LLM Changelog
 
+## [0.3.15] - 2026-03-21
+
+### Changed
+- Pin ruby_llm dependency from `>= 1.0` to `~> 1.13` to prevent breaking changes from a future 2.0 release
+
+## [0.3.14] - 2026-03-21
+
+### Added
+- `Legion::LLM::Arbitrage` module for cost-aware model selection: configurable cost table (per-1M-token input/output prices), `cheapest_for(capability:, max_cost:)` filters eligible models and returns the cheapest, `estimated_cost` for per-request USD estimates, settings-defined cost_table overrides, quality_floor and capability-tier filtering
+- `Legion::LLM::Batch` module for non-urgent request batching: `enqueue` stores requests in an in-process queue with UUID tracking, `flush` groups by provider/model and invokes callbacks, configurable window_seconds and max_batch_size, `reset!` for test isolation
+- `Legion::LLM::Scheduling` module for off-peak deferral: `should_defer?(intent:, urgency:)` checks configurable peak hours and intent eligibility, `peak_hours?` evaluates UTC hour against configurable range, `next_off_peak` returns next off-peak window capped at max_defer_hours
+- Default settings for all three features under `llm.arbitrage`, `llm.batch`, `llm.scheduling` — all disabled by default (opt-in)
+- 3 new spec files: `arbitrage_spec.rb` (18 examples), `batch_spec.rb` (16 examples), `scheduling_spec.rb` (24 examples)
+
 ## [0.3.13] - 2026-03-21
 
 ### Added

data/CODEOWNERS

@@ -0,0 +1 @@
+* @Esity

data/legion-llm.gemspec

@@ -30,6 +30,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'lex-claude'
   spec.add_dependency 'lex-gemini'
   spec.add_dependency 'lex-openai'
-  spec.add_dependency 'ruby_llm', '>= 1.0'
+  spec.add_dependency 'ruby_llm', '~> 1.13'
   spec.add_dependency 'tzinfo', '>= 2.0'
 end

data/lib/legion/llm/arbitrage.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Arbitrage
+      # Default cost table: per-1M-token input/output prices in USD.
+      # Overridable via settings: llm.arbitrage.cost_table
+      DEFAULT_COST_TABLE = {
+        'claude-sonnet-4-6'                 => { input: 3.0, output: 15.0 },
+        'us.anthropic.claude-sonnet-4-6-v1' => { input: 3.0, output: 15.0 },
+        'gpt-4o'                            => { input: 2.5,  output: 10.0 },
+        'gpt-4o-mini'                       => { input: 0.15, output: 0.60 },
+        'gemini-2.0-flash'                  => { input: 0.10, output: 0.40 },
+        'llama3'                            => { input: 0.0,  output: 0.0  }
+      }.freeze
+
+      class << self
+        # Returns true when arbitrage is enabled in settings.
+        def enabled?
+          settings.fetch(:enabled, false) == true
+        end
+
+        # Returns the estimated cost for a request with the given token counts.
+        #
+        # @param model [String] model identifier
+        # @param input_tokens [Integer] estimated number of input tokens
+        # @param output_tokens [Integer] estimated number of output tokens
+        # @return [Float, nil] estimated cost in USD, or nil if model not in table
+        def estimated_cost(model:, input_tokens: 1000, output_tokens: 500)
+          entry = cost_table[model.to_s]
+          return nil if entry.nil?
+
+          ((entry[:input] * input_tokens) + (entry[:output] * output_tokens)) / 1_000_000.0
+        end
+
+        # Selects the cheapest model that meets the capability and quality floor requirements.
+        #
+        # @param capability [String, Symbol] required capability tier (e.g., :basic, :moderate, :reasoning)
+        # @param max_cost [Float, nil] maximum acceptable cost per typical request (USD); nil means no limit
+        # @param input_tokens [Integer] estimated input tokens for cost calculation
+        # @param output_tokens [Integer] estimated output tokens for cost calculation
+        # @return [String, nil] cheapest eligible model ID, or nil if none qualify
+        def cheapest_for(capability: :moderate, max_cost: nil, input_tokens: 1000, output_tokens: 500)
+          return nil unless enabled?
+
+          quality_floor = settings.fetch(:quality_floor, 0.7)
+          eligible = eligible_models(capability: capability, _quality_floor: quality_floor)
+
+          scored = eligible.filter_map do |model|
+            cost = estimated_cost(model: model, input_tokens: input_tokens, output_tokens: output_tokens)
+            next if cost.nil?
+            next if max_cost && cost > max_cost
+
+            [model, cost]
+          end
+
+          return nil if scored.empty?
+
+          scored.min_by { |_model, cost| cost }&.first
+        end
+
+        # Returns the merged cost table: defaults overridden by any settings-defined entries.
+        def cost_table
+          overrides = settings.fetch(:cost_table, {})
+          return DEFAULT_COST_TABLE if overrides.nil? || overrides.empty?
+
+          merged = DEFAULT_COST_TABLE.dup
+          overrides.each do |model, costs|
+            entry = costs.transform_keys(&:to_sym)
+            merged[model.to_s] = entry
+          end
+          merged
+        end
+
+        private
+
+        def settings
+          llm = Legion::Settings[:llm]
+          return {} unless llm.is_a?(Hash)
+
+          arb = llm[:arbitrage] || llm['arbitrage'] || {}
+          arb.is_a?(Hash) ? arb.transform_keys(&:to_sym) : {}
+        rescue StandardError
+          {}
+        end
+
+        # Returns models eligible for the given capability tier based on quality floor.
+        # The quality floor maps capability tiers to minimum acceptable quality scores (0.0-1.0).
+        # Models that are local (cost 0) always qualify for :basic capability.
+        def eligible_models(capability:, _quality_floor: 0.7)
+          cap = capability.to_sym
+
+          # Capability tiers determine which models are semantically appropriate.
+          # :reasoning requires frontier models; :basic allows cheap/local models.
+          # _quality_floor reserved for future scoring integration.
+          disqualified_for_reasoning = %w[gpt-4o-mini gemini-2.0-flash llama3]
+
+          cost_table.keys.reject do |model|
+            cap == :reasoning && disqualified_for_reasoning.include?(model)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/batch.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Legion
+  module LLM
+    module Batch
+      class << self
+        # Returns true when request batching is enabled in settings.
+        def enabled?
+          settings.fetch(:enabled, false) == true
+        end
+
+        # Enqueues a request for deferred batch processing.
+        #
+        # @param messages [Array<Hash>] chat messages array
+        # @param model    [String]      model to use
+        # @param provider [Symbol, nil] provider override
+        # @param callback [Proc, nil]   called with result hash when batch is flushed
+        # @param priority [Symbol]      :normal or :low (informational only)
+        # @param opts     [Hash]        additional options forwarded to provider
+        # @return [String] batch_request_id
+        def enqueue(messages:, model:, callback: nil, provider: nil, priority: :normal, **opts)
+          request_id = SecureRandom.uuid
+
+          entry = {
+            id:        request_id,
+            messages:  messages,
+            model:     model,
+            provider:  provider,
+            callback:  callback,
+            priority:  priority,
+            opts:      opts,
+            queued_at: Time.now.utc
+          }
+
+          queue << entry
+          Legion::Logging.debug "Legion::LLM::Batch enqueued #{request_id} (queue size: #{queue.size})"
+          request_id
+        end
+
+        # Flushes accumulated requests up to max_size.
+        # Groups entries by provider+model and invokes callbacks with a stub result.
+        # In production this would submit to provider batch APIs; here it logs and returns
+        # per-request result hashes for callback delivery.
+        #
+        # @param max_size [Integer] maximum number of requests to flush in one pass
+        # @param max_wait [Integer] only flush entries older than this many seconds (0 = all)
+        # @return [Array<Hash>] array of { id:, status:, result: } hashes
+        def flush(max_size: nil, max_wait: nil)
+          effective_max  = max_size || settings.fetch(:max_batch_size, 100)
+          effective_wait = max_wait || settings.fetch(:window_seconds, 300)
+
+          cutoff = Time.now.utc - effective_wait
+          to_flush = queue.select { |e| e[:queued_at] <= cutoff }.first(effective_max)
+
+          return [] if to_flush.empty?
+
+          to_flush.each { |e| queue.delete(e) }
+          Legion::Logging.debug "Legion::LLM::Batch flushing #{to_flush.size} request(s)"
+
+          groups = to_flush.group_by { |e| [e[:provider], e[:model]] }
+          results = []
+
+          groups.each do |(provider, model), entries|
+            entries.each do |entry|
+              result = submit_single(entry, provider: provider, model: model)
+              entry[:callback]&.call(result)
+              results << { id: entry[:id], status: result[:status], result: result }
+            end
+          end
+
+          results
+        end
+
+        # Returns the current number of requests in the queue.
+        def queue_size
+          queue.size
+        end
+
+        # Clears the queue (useful for testing).
+        def reset!
+          @queue = []
+        end
+
+        private
+
+        def queue
+          @queue ||= []
+        end
+
+        def settings
+          llm = Legion::Settings[:llm]
+          return {} unless llm.is_a?(Hash)
+
+          b = llm[:batch] || llm['batch'] || {}
+          b.is_a?(Hash) ? b.transform_keys(&:to_sym) : {}
+        rescue StandardError
+          {}
+        end
+
+        def submit_single(entry, provider:, model:)
+          {
+            status:   :batched,
+            model:    model,
+            provider: provider,
+            id:       entry[:id],
+            response: nil,
+            meta:     { batched: true, queued_at: entry[:queued_at] }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/scheduling.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Scheduling
+      # Default peak hours in UTC: 14:00-22:00 (9 AM - 5 PM CT)
+      DEFAULT_PEAK_RANGE = (14..22)
+
+      # Intents that are eligible for deferral during peak hours.
+      DEFAULT_DEFER_INTENTS = %i[batch background maintenance].freeze
+
+      class << self
+        # Returns true when off-peak scheduling is enabled in settings.
+        def enabled?
+          settings.fetch(:enabled, false) == true
+        end
+
+        # Determines whether a request should be deferred to off-peak hours.
+        #
+        # @param intent  [Symbol, String] the request intent
+        # @param urgency [Symbol]         :immediate bypasses deferral regardless of settings
+        # @return [Boolean]
+        def should_defer?(intent: :normal, urgency: :normal)
+          return false unless enabled?
+          return false if urgency.to_sym == :immediate
+
+          eligible_for_deferral?(intent.to_sym) && peak_hours?
+        end
+
+        # Returns true if the current UTC hour falls within the configured peak window.
+        def peak_hours?
+          hour = Time.now.utc.hour
+          peak_range.cover?(hour)
+        end
+
+        # Returns the next off-peak time as a Time object (UTC).
+        # Off-peak begins at the hour after the peak window ends.
+        #
+        # @return [Time] next off-peak start time
+        def next_off_peak
+          now = Time.now.utc
+          peak_end = peak_range.last
+          max_defer = settings.fetch(:max_defer_hours, 8)
+
+          next_time = if now.hour < peak_range.first
+                        # Before peak — off-peak is now
+                        now
+                      else
+                        # During or after peak — next off-peak is at peak_end + 1
+                        candidate = Time.utc(now.year, now.month, now.day, peak_end + 1, 0, 0)
+                        candidate += 86_400 if candidate <= now
+                        candidate
+                      end
+
+          # Cap at max_defer_hours from now
+          cap = now + (max_defer * 3600)
+          [next_time, cap].min
+        end
+
+        private
+
+        def settings
+          llm = Legion::Settings[:llm]
+          return {} unless llm.is_a?(Hash)
+
+          s = llm[:scheduling] || llm['scheduling'] || {}
+          s.is_a?(Hash) ? s.transform_keys(&:to_sym) : {}
+        rescue StandardError
+          {}
+        end
+
+        def peak_range
+          raw = settings[:peak_hours_utc]
+          return DEFAULT_PEAK_RANGE unless raw.is_a?(String) && raw.include?('-')
+
+          parts = raw.split('-')
+          return DEFAULT_PEAK_RANGE unless parts.size == 2
+
+          start_h = Integer(parts[0], 10)
+          end_h   = Integer(parts[1], 10)
+          (start_h..end_h)
+        rescue ArgumentError
+          DEFAULT_PEAK_RANGE
+        end
+
+        def defer_intents
+          raw = settings[:defer_intents]
+          return DEFAULT_DEFER_INTENTS unless raw.is_a?(Array)
+
+          raw.map { |i| i.to_s.to_sym }
+        end
+
+        def eligible_for_deferral?(intent)
+          defer_intents.include?(intent)
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/settings.rb

@@ -15,7 +15,10 @@ module Legion
           discovery:        discovery_defaults,
           gateway:          gateway_defaults,
           daemon:           daemon_defaults,
-          prompt_caching:   prompt_caching_defaults
+          prompt_caching:   prompt_caching_defaults,
+          arbitrage:        arbitrage_defaults,
+          batch:            batch_defaults,
+          scheduling:       scheduling_defaults
         }
       end
 
@@ -81,6 +84,34 @@ module Legion
         }
       end
 
+      def self.arbitrage_defaults
+        {
+          enabled:            false,
+          prefer_cheapest:    true,
+          quality_floor:      0.7,
+          cost_table_refresh: 86_400,
+          cost_table:         {}
+        }
+      end
+
+      def self.batch_defaults
+        {
+          enabled:          false,
+          window_seconds:   300,
+          max_batch_size:   100,
+          eligible_intents: %w[batch background low_priority]
+        }
+      end
+
+      def self.scheduling_defaults
+        {
+          enabled:         false,
+          peak_hours_utc:  '14-22',
+          defer_intents:   %w[batch background],
+          max_defer_hours: 8
+        }
+      end
+
       def self.providers
         {
           bedrock:   {

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.3.13'
+    VERSION = '0.3.15'
   end
 end

data/lib/legion/llm.rb

@@ -12,6 +12,9 @@ require 'legion/llm/hooks'
 require 'legion/llm/cache'
 require_relative 'llm/response_cache'
 require_relative 'llm/daemon_client'
+require_relative 'llm/arbitrage'
+require_relative 'llm/batch'
+require_relative 'llm/scheduling'
 
 begin
   require 'legion/extensions/llm/gateway'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.15
 platform: ruby
 authors:
 - Esity
@@ -83,16 +83,16 @@ dependencies:
   name: ruby_llm
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +123,7 @@ files:
 - ".rubocop.yml"
 - CHANGELOG.md
 - CLAUDE.md
+- CODEOWNERS
 - Gemfile
 - LICENSE
 - README.md
@@ -130,6 +131,8 @@ files:
 - docs/plans/2026-03-15-ollama-discovery-implementation.md
 - legion-llm.gemspec
 - lib/legion/llm.rb
+- lib/legion/llm/arbitrage.rb
+- lib/legion/llm/batch.rb
 - lib/legion/llm/bedrock_bearer_auth.rb
 - lib/legion/llm/cache.rb
 - lib/legion/llm/claude_config_loader.rb
@@ -152,6 +155,7 @@ files:
 - lib/legion/llm/router/health_tracker.rb
 - lib/legion/llm/router/resolution.rb
 - lib/legion/llm/router/rule.rb
+- lib/legion/llm/scheduling.rb
 - lib/legion/llm/settings.rb
 - lib/legion/llm/shadow_eval.rb
 - lib/legion/llm/structured_output.rb