legion-llm 0.3.1

data/lib/legion/llm/router/health_tracker.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Router
+      class HealthTracker
+        OPEN_PENALTY         = -50
+        LATENCY_THRESHOLD_MS = 5000
+        LATENCY_PENALTY_STEP = -10
+
+        def initialize(window_seconds: 300, failure_threshold: 3, cooldown_seconds: 60)
+          @window_seconds    = window_seconds
+          @failure_threshold = failure_threshold
+          @cooldown_seconds  = cooldown_seconds
+
+          @circuits       = {}
+          @latency_window = {}
+          @handlers       = {}
+          @mutex          = Mutex.new
+
+          register_default_handlers
+        end
+
+        # Register a custom handler for a signal type.
+        def register_handler(signal, &block)
+          @handlers[signal.to_sym] = block
+        end
+
+        # Thread-safe signal intake. Dispatches to the registered handler if one exists.
+        def report(provider:, signal:, value:, metadata: {})
+          sym     = signal.to_sym
+          handler = @handlers[sym]
+          return nil unless handler
+
+          payload = { provider: provider, signal: sym, value: value, metadata: metadata, at: Time.now }
+          @mutex.synchronize { handler.call(payload) }
+        end
+
+        # Returns total priority adjustment for a provider.
+        # Combines circuit-breaker penalty and latency penalty.
+        def adjustment(provider)
+          circuit_adjustment(provider) + latency_adjustment(provider)
+        end
+
+        # Returns :closed, :open, or :half_open.
+        def circuit_state(provider)
+          circuit = @circuits[provider]
+          return :closed if circuit.nil?
+
+          if circuit[:state] == :open
+            elapsed = Time.now - circuit[:opened_at]
+            return :half_open if elapsed >= @cooldown_seconds
+          end
+
+          circuit[:state]
+        end
+
+        # Clears circuit and latency data for a single provider.
+        def reset(provider)
+          @mutex.synchronize do
+            @circuits.delete(provider)
+            @latency_window.delete(provider)
+          end
+        end
+
+        # Clears all state.
+        def reset_all
+          @mutex.synchronize do
+            @circuits.clear
+            @latency_window.clear
+          end
+        end
+
+        private
+
+        def register_default_handlers
+          register_handler(:error) do |payload|
+            provider = payload[:provider]
+            ensure_circuit(provider)
+            circuit = @circuits[provider]
+
+            if circuit_state(provider) == :half_open
+              circuit[:state]     = :open
+              circuit[:opened_at] = Time.now
+            else
+              circuit[:failures] += 1.0
+              if circuit[:failures] >= @failure_threshold
+                circuit[:state]     = :open
+                circuit[:opened_at] = Time.now
+              end
+            end
+          end
+
+          register_handler(:success) do |payload|
+            provider = payload[:provider]
+            ensure_circuit(provider)
+            circuit             = @circuits[provider]
+            circuit[:failures]  = 0
+            circuit[:state]     = :closed
+            circuit[:opened_at] = nil
+          end
+
+          register_handler(:quality_failure) do |payload|
+            provider = payload[:provider]
+            ensure_circuit(provider)
+            circuit = @circuits[provider]
+
+            if circuit_state(provider) == :half_open
+              circuit[:state]     = :open
+              circuit[:opened_at] = Time.now
+            else
+              circuit[:failures] += 0.5
+              if circuit[:failures] >= @failure_threshold
+                circuit[:state]     = :open
+                circuit[:opened_at] = Time.now
+              end
+            end
+          end
+
+          register_handler(:latency) do |payload|
+            provider = payload[:provider]
+            @latency_window[provider] ||= []
+            @latency_window[provider] << { value: payload[:value], at: payload[:at] }
+          end
+        end
+
+        def ensure_circuit(provider)
+          @circuits[provider] ||= { state: :closed, failures: 0.0, opened_at: nil }
+        end
+
+        def circuit_adjustment(provider)
+          case circuit_state(provider)
+          when :open      then OPEN_PENALTY
+          when :half_open then OPEN_PENALTY / 2
+          else                 0
+          end
+        end
+
+        def latency_adjustment(provider)
+          entries = @latency_window[provider]
+          return 0 if entries.nil? || entries.empty?
+
+          cutoff = Time.now - @window_seconds
+          recent = entries.select { |e| e[:at] >= cutoff }
+
+          # Prune stale entries in-place
+          @latency_window[provider] = recent
+
+          return 0 if recent.empty?
+
+          avg = recent.sum { |e| e[:value] } / recent.size.to_f
+          return 0 if avg <= LATENCY_THRESHOLD_MS
+
+          multiplier = (avg / LATENCY_THRESHOLD_MS).floor
+          [LATENCY_PENALTY_STEP * multiplier, OPEN_PENALTY].max
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/router/resolution.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Router
+      class Resolution
+        attr_reader :tier, :provider, :model, :rule, :metadata, :compress_level
+
+        def initialize(tier:, provider:, model:, rule: nil, metadata: {}, compress_level: 0)
+          @tier           = tier.to_sym
+          @provider       = provider.to_sym
+          @model          = model
+          @rule           = rule
+          @metadata       = metadata
+          @compress_level = compress_level.to_i
+        end
+
+        def local?
+          @tier == :local
+        end
+
+        def fleet?
+          @tier == :fleet
+        end
+
+        def cloud?
+          @tier == :cloud
+        end
+
+        def to_h
+          {
+            tier:           @tier,
+            provider:       @provider,
+            model:          @model,
+            rule:           @rule,
+            metadata:       @metadata,
+            compress_level: @compress_level
+          }
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/router/rule.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'resolution'
+
+module Legion
+  module LLM
+    module Router
+      class Rule
+        attr_reader :name, :conditions, :target, :priority, :constraint, :fallback, :cost_multiplier, :schedule, :note
+
+        def self.from_hash(hash)
+          h = hash.transform_keys(&:to_sym)
+          new(
+            name:            h[:name],
+            conditions:      h[:when]  || {},
+            target:          h[:then]  || {},
+            priority:        h.fetch(:priority, 0),
+            constraint:      h[:constraint],
+            fallback:        h[:fallback].is_a?(Hash) ? h[:fallback].transform_keys(&:to_sym) : h[:fallback]&.to_sym,
+            cost_multiplier: h.fetch(:cost_multiplier, 1.0),
+            schedule:        h[:schedule],
+            note:            h[:note]
+          )
+        end
+
+        def initialize(name:, conditions:, target:, priority: 0, constraint: nil, fallback: nil,
+                       cost_multiplier: 1.0, schedule: nil, note: nil)
+          @name            = name
+          @conditions      = conditions.transform_keys(&:to_sym)
+          @target          = target.transform_keys(&:to_sym)
+          @priority        = priority
+          @constraint      = constraint
+          @fallback        = fallback
+          @cost_multiplier = cost_multiplier
+          @schedule        = schedule
+          @note            = note
+        end
+
+        def matches_intent?(intent)
+          @conditions.all? do |key, value|
+            return false unless intent.key?(key)
+
+            intent[key].to_s == value.to_s
+          end
+        end
+
+        def to_resolution
+          target_without_compress = @target.except(:compress_level)
+          Resolution.new(
+            **target_without_compress,
+            rule:           @name,
+            metadata:       { cost_multiplier: @cost_multiplier, fallback: @fallback }.compact,
+            compress_level: @target.fetch(:compress_level, 0)
+          )
+        end
+
+        def within_schedule?(now = Time.now)
+          return true if @schedule.nil? || (@schedule.respond_to?(:empty?) && @schedule.empty?)
+
+          sched = @schedule.transform_keys(&:to_s)
+          now = localize(now, sched['timezone'])
+
+          return false if sched['valid_from']  && now < Time.parse(sched['valid_from'])
+          return false if sched['valid_until'] && now > Time.parse(sched['valid_until'])
+          return false if sched['hours'] && !within_hours?(sched['hours'], now)
+          return false if sched['days']  && !on_allowed_day?(sched['days'], now)
+
+          true
+        end
+
+        private
+
+        def localize(time, timezone_name)
+          return time unless timezone_name
+
+          require 'tzinfo'
+          TZInfo::Timezone.get(timezone_name).to_local(time)
+        end
+
+        def within_hours?(ranges, now)
+          current = (now.hour * 60) + now.min
+          ranges.any? do |range|
+            start_str, end_str = range.split('-')
+            start_min = time_str_to_minutes(start_str)
+            end_min   = time_str_to_minutes(end_str)
+            current.between?(start_min, end_min)
+          end
+        end
+
+        def on_allowed_day?(days, now)
+          today = now.strftime('%A').downcase
+          days.map { |d| d.to_s.downcase }.include?(today)
+        end
+
+        def time_str_to_minutes(str)
+          parts = str.split(':')
+          (parts[0].to_i * 60) + parts[1].to_i
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/router.rb

@@ -0,0 +1,279 @@
+# frozen_string_literal: true
+
+require_relative 'router/resolution'
+require_relative 'router/rule'
+require_relative 'router/health_tracker'
+require_relative 'router/escalation_chain'
+require_relative 'discovery/ollama'
+require_relative 'discovery/system'
+
+module Legion
+  module LLM
+    module Router
+      class << self
+        # Resolve an LLM routing intent to a tier/provider/model decision.
+        #
+        # @param intent   [Hash, nil] routing intent (capability, privacy, etc.)
+        # @param tier     [Symbol, nil] explicit tier override — skips rule matching
+        # @param model    [String, nil] explicit model override
+        # @param provider [Symbol, nil] explicit provider override
+        # @return [Resolution, nil]
+        def resolve(intent: nil, tier: nil, model: nil, provider: nil)
+          return explicit_resolution(tier, provider, model) if tier
+
+          return nil unless routing_enabled? && intent
+
+          merged = merge_defaults(intent)
+          rules = load_rules
+          candidates = select_candidates(rules, merged)
+          best = pick_best(candidates)
+          best&.to_resolution
+        end
+
+        def resolve_chain(intent: nil, tier: nil, model: nil, provider: nil, max_escalations: nil)
+          max = max_escalations || escalation_max_attempts
+          return chain_from_defaults(model, provider, max) unless routing_enabled? && (intent || tier)
+          return EscalationChain.new(resolutions: [explicit_resolution(tier, provider, model)], max_attempts: max) if tier
+
+          chain_from_intent(intent, max)
+        end
+
+        def health_tracker
+          @health_tracker ||= build_health_tracker
+        end
+
+        def routing_enabled?
+          settings = routing_settings
+          return false if settings.nil? || settings.empty?
+          return false unless settings[:enabled]
+
+          rules = settings[:rules]
+          rules.is_a?(Array) && !rules.empty?
+        end
+
+        def reset!
+          @health_tracker = nil
+        end
+
+        # Check whether a tier can be used right now.
+        # :local — always available
+        # :fleet — available when Legion::Transport is loaded
+        # :cloud — always available
+        def tier_available?(tier)
+          return Legion.const_defined?('Transport') if tier.to_sym == :fleet
+
+          true
+        end
+
+        private
+
+        def explicit_resolution(tier, provider, model)
+          resolved_provider = provider ? provider.to_sym : default_provider_for_tier(tier)
+          resolved_model = model || default_model_for_tier(tier)
+
+          Resolution.new(
+            tier:     tier,
+            provider: resolved_provider,
+            model:    resolved_model,
+            rule:     'explicit'
+          )
+        end
+
+        def merge_defaults(intent)
+          defaults = (routing_settings[:default_intent] || {})
+                     .transform_keys(&:to_sym)
+                     .transform_values { |v| v.respond_to?(:to_sym) ? v.to_sym : v }
+
+          normalized_intent = intent
+                              .transform_keys(&:to_sym)
+                              .transform_values { |v| v.respond_to?(:to_sym) ? v.to_sym : v }
+
+          defaults.merge(normalized_intent)
+        end
+
+        def load_rules
+          raw = routing_settings[:rules] || []
+          raw.map { |h| Rule.from_hash(h.transform_keys(&:to_sym)) }
+        end
+
+        def select_candidates(rules, intent)
+          # 1. Collect constraints from constraint rules that match the intent
+          constraints = rules
+                        .select { |r| r.constraint && r.matches_intent?(intent) }
+                        .map(&:constraint)
+
+          # 2. Filter by intent match
+          matched = rules.select { |r| r.matches_intent?(intent) }
+
+          # 3. Filter by schedule
+          scheduled = matched.select(&:within_schedule?)
+
+          # 4. Reject rules excluded by active constraints
+          unconstrained = scheduled.reject { |r| excluded_by_constraint?(r, constraints) }
+
+          # 4.5 Reject Ollama rules where model is not pulled or doesn't fit
+          discovered = unconstrained.reject { |r| excluded_by_discovery?(r) }
+
+          # 5. Filter by tier availability
+          discovered.select { |r| tier_available?(r.target[:tier] || r.target['tier']) }
+        end
+
+        def excluded_by_constraint?(rule, constraints)
+          return false if constraints.empty?
+
+          tier = (rule.target[:tier] || rule.target['tier'])&.to_sym
+
+          constraints.any? do |c|
+            c.to_s == 'never_cloud' && tier == :cloud
+          end
+        end
+
+        def excluded_by_discovery?(rule)
+          return false unless discovery_enabled?
+
+          tier     = (rule.target[:tier] || rule.target['tier'])&.to_sym
+          provider = (rule.target[:provider] || rule.target['provider'])&.to_sym
+          model    = rule.target[:model] || rule.target['model']
+
+          return false unless tier == :local && provider == :ollama && model
+
+          return true unless Discovery::Ollama.model_available?(model)
+
+          model_bytes = Discovery::Ollama.model_size(model)
+          available   = Discovery::System.available_memory_mb
+          return false if model_bytes.nil? || available.nil?
+
+          floor = discovery_settings[:memory_floor_mb] || 2048
+          model_mb = model_bytes / 1024 / 1024
+          model_mb > (available - floor)
+        end
+
+        def discovery_enabled?
+          ds = discovery_settings
+          ds.fetch(:enabled, true)
+        end
+
+        def discovery_settings
+          llm = Legion::Settings[:llm]
+          return {} unless llm.is_a?(Hash)
+
+          (llm[:discovery] || {}).transform_keys(&:to_sym)
+        rescue StandardError
+          {}
+        end
+
+        def pick_best(candidates)
+          return nil if candidates.empty?
+
+          candidates.max_by { |r| effective_priority(r) }
+        end
+
+        def effective_priority(rule)
+          provider   = (rule.target[:provider] || rule.target['provider'])&.to_sym
+          cost_bonus = (1.0 - rule.cost_multiplier) * 10
+          rule.priority + health_tracker.adjustment(provider) + cost_bonus
+        end
+
+        def routing_settings
+          llm = Legion::Settings[:llm]
+          return {} unless llm.is_a?(Hash)
+
+          routing = llm[:routing] || llm['routing'] || {}
+          routing.transform_keys(&:to_sym)
+        end
+
+        def build_health_tracker
+          settings = routing_settings
+          cb       = (settings[:circuit_breaker] || {}).transform_keys(&:to_sym)
+
+          HealthTracker.new(
+            window_seconds:    settings.fetch(:window_seconds, 300),
+            failure_threshold: cb.fetch(:failure_threshold, 3),
+            cooldown_seconds:  cb.fetch(:cooldown_seconds, 60)
+          )
+        end
+
+        def default_provider_for_tier(tier)
+          if tier.to_sym == :cloud
+            default = routing_settings[:default_provider]
+            default ? default.to_sym : :bedrock
+          else
+            :ollama
+          end
+        end
+
+        def default_model_for_tier(tier)
+          case tier.to_sym
+          when :local
+            ollama = Legion::Settings[:llm].dig(:providers, :ollama) || {}
+            ollama[:default_model] || 'llama3'
+          when :fleet then 'llama4:70b'
+          when :cloud
+            Legion::Settings[:llm][:default_model] || 'claude-sonnet-4-6'
+          else 'llama3'
+          end
+        end
+
+        def chain_from_defaults(model, provider, max)
+          fallback_model    = model || default_settings_model
+          fallback_provider = (provider || default_settings_provider)&.to_sym
+          res = Resolution.new(tier: :cloud, provider: fallback_provider || :bedrock, model: fallback_model || 'claude-sonnet-4-6')
+          EscalationChain.new(resolutions: [res], max_attempts: max)
+        end
+
+        def chain_from_intent(intent, max)
+          merged     = intent ? merge_defaults(intent) : {}
+          rules      = load_rules
+          candidates = select_candidates(rules, merged)
+          sorted     = candidates.sort_by { |r| -effective_priority(r) }
+          resolutions = sorted.map(&:to_resolution)
+          resolutions = build_fallback_chain(sorted.first, sorted, resolutions) if sorted.first&.fallback
+          resolutions = resolutions.uniq { |r| [r.provider, r.model] }
+          EscalationChain.new(resolutions: resolutions, max_attempts: max)
+        end
+
+        def build_fallback_chain(primary_rule, candidates, default_chain)
+          chain = [primary_rule.to_resolution]
+          current = primary_rule
+
+          while current.fallback
+            fallback_target = current.fallback
+            if fallback_target.is_a?(Hash)
+              fb = fallback_target.transform_keys(&:to_sym)
+              fb_tier     = fb[:tier]&.to_sym || :cloud
+              fb_provider = fb[:provider]&.to_sym || default_provider_for_tier(fb_tier)
+              fb_model    = fb[:model] || default_model_for_tier(fb_tier)
+              chain << Resolution.new(tier: fb_tier, provider: fb_provider, model: fb_model)
+              break
+            else
+              next_rule = candidates.find { |r| r.name == fallback_target.to_s }
+              break unless next_rule
+
+              chain << next_rule.to_resolution
+              current = next_rule
+            end
+          end
+
+          remaining = default_chain.reject { |r| chain.any? { |c| c.provider == r.provider && c.model == r.model } }
+          chain + remaining
+        end
+
+        def escalation_max_attempts
+          settings = routing_settings
+          esc = (settings[:escalation] || {}).transform_keys(&:to_sym)
+          esc.fetch(:max_attempts, 3)
+        end
+
+        def default_settings_model
+          llm = Legion::Settings[:llm]
+          llm[:default_model] if llm.is_a?(Hash)
+        end
+
+        def default_settings_provider
+          llm = Legion::Settings[:llm]
+          llm[:default_provider] if llm.is_a?(Hash)
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/settings.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Settings
+      def self.default
+        {
+          enabled:          true,
+          connected:        false,
+          default_model:    nil,
+          default_provider: nil,
+          providers:        providers,
+          routing:          routing_defaults,
+          discovery:        discovery_defaults
+        }
+      end
+
+      def self.discovery_defaults
+        {
+          enabled:         true,
+          refresh_seconds: 60,
+          memory_floor_mb: 2048
+        }
+      end
+
+      def self.routing_defaults
+        {
+          enabled:        false,
+          default_intent: { privacy: 'normal', capability: 'moderate', cost: 'normal' },
+          tiers:          {
+            local: { provider: 'ollama' },
+            fleet: { queue: 'llm.inference', timeout_seconds: 30 },
+            cloud: { providers: %w[bedrock anthropic] }
+          },
+          health:         {
+            window_seconds:               300,
+            circuit_breaker:              { failure_threshold: 3, cooldown_seconds: 60 },
+            latency_penalty_threshold_ms: 5000,
+            budget:                       { daily_limit_usd: nil, monthly_limit_usd: nil }
+          },
+          escalation:     {
+            enabled:           false,
+            max_attempts:      3,
+            quality_threshold: 50
+          },
+          rules:          []
+        }
+      end
+
+      def self.providers
+        {
+          bedrock:   {
+            enabled:       false,
+            default_model: 'us.anthropic.claude-sonnet-4-6-v1',
+            api_key:       nil,
+            secret_key:    nil,
+            session_token: nil,
+            bearer_token:  nil,
+            region:        'us-east-2'
+          },
+          anthropic: {
+            enabled:       false,
+            default_model: 'claude-sonnet-4-6',
+            api_key:       nil
+          },
+          openai:    {
+            enabled:       false,
+            default_model: 'gpt-4o',
+            api_key:       nil
+          },
+          gemini:    {
+            enabled:       false,
+            default_model: 'gemini-2.0-flash',
+            api_key:       nil
+          },
+          ollama:    {
+            enabled:       false,
+            default_model: 'llama3',
+            base_url:      'http://localhost:11434'
+          }
+        }
+      end
+    end
+  end
+end
+
+begin
+  Legion::Settings.merge_settings('llm', Legion::LLM::Settings.default) if Legion.const_defined?('Settings')
+rescue StandardError => e
+  if Legion.const_defined?('Logging') && Legion::Logging.respond_to?(:fatal)
+    Legion::Logging.fatal(e.message)
+    Legion::Logging.fatal(e.backtrace)
+  else
+    puts e.message
+    puts e.backtrace
+  end
+end

data/lib/legion/llm/transport/exchanges/escalation.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Legion
+  module LLM
+    module Transport
+      module Exchanges
+        class Escalation < ::Legion::Transport::Exchange
+          exchange_name 'llm.escalation'
+          exchange_type :topic
+        end
+      end
+    end
+  end
+end