legion-llm 0.6.7 → 0.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7af59b9f91a1e3c4cb4d6d03758dd927695aba9cbe4f9ac52f221b100a40294
-  data.tar.gz: 53ad50449a3f984a1be1faeecb68ec35a72da3b4f691811395787c7ba4343a80
+  metadata.gz: 426b32ea868afbd22ddbe70fb975c8b684b5afc306bd8e2efa820c5f01819fff
+  data.tar.gz: 31dec2079d806c124824eac38098bd5642391547dfa53b88b1b8e4c0f657a693
 SHA512:
-  metadata.gz: 9f4b5a94917ef6e131769128212860363a51ae893feb9dbe7c5b53f3697a15a78680436d152e294f5b7498cf59a167e973c98307df896186dedc83feb9614b5a
-  data.tar.gz: 4621025b19339b1e627da7b5338b034c8c459090a29c539a409aeea80549bac8a5797d314ff6e4b9715eebd9d35941cbce2f0f4d25e6f1a6b5a40bc815a7e0bb
+  metadata.gz: 36d79c94a91d353e6cff51c4a23a082f8e21b7d886edca017f2cfa13d7d292495aa9375007d781a056e98dcffc1b1fda0290affbb88387dc028dacbb35ee1fb1
+  data.tar.gz: 4a0c36734d64cd7a86efa760d0fd932a542f52823cdb3096ffd099808274d77a0c8f7cc569abe1edc2ee0f3cf3f084a90ab040c88991bd9c4b842c612830ba4d

data/CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## [Unreleased]
 
+## [0.6.8] - 2026-04-01
+
+### Added
+- `ASYNC_SAFE_STEPS` constant and async dispatch for fire-and-forget post-provider pipeline steps (`post_response`, `knowledge_capture`, `response_return`) via a shared `Concurrent::FixedThreadPool` — frees Puma threads immediately after response normalization; `context_store` continues to run synchronously
+- `pipeline_async_post_steps` setting (default `true`) controls sync vs async post-step dispatch
+- Cached embed provider fallback chain at startup (`build_embedding_fallback_chain`, `provider_enabled?`) — stops re-probing on every embed failure
+- Hard gate on disabled providers in `Embeddings.generate` and `generate_batch` — providers with `enabled: false` are never tried
+
+### Changed
+- `embedding_defaults[:provider_fallback]` no longer includes `azure` (leaked via `deep_merge` array concat)
+- `find_fallback_provider` walks the cached startup chain instead of re-probing live
+
 ## [0.6.7] - 2026-04-01
 
 ### Added

data/lib/legion/llm/embeddings.rb

@@ -34,8 +34,10 @@ module Legion
           return { vector: nil, model: model, provider: provider, error: 'LLM not started' } unless LLM.started?
 
           provider ||= resolve_provider
-          model    ||= resolve_model(provider)
-          text       = apply_prefix(text, model: model, task: task)
+          return { vector: nil, model: model, provider: provider, error: "provider #{provider} is disabled" } if provider_disabled?(provider)
+
+          model ||= resolve_model(provider)
+          text    = apply_prefix(text, model: model, task: task)
 
           return generate_ollama(text: text, model: model) if provider&.to_sym == :ollama
           return generate_azure(text: text, model: model, dimensions: dimensions) if provider&.to_sym == :azure
@@ -54,8 +56,11 @@ module Legion
           return texts.map { |_| { vector: nil, error: 'LLM not started' } } unless LLM.started?
 
           provider ||= resolve_provider
-          model    ||= resolve_model(provider)
-          texts      = texts.map { |t| apply_prefix(t, model: model, task: task) }
+          disabled_result = disabled_batch_result(texts, provider, model)
+          return disabled_result if disabled_result
+
+          model  ||= resolve_model(provider)
+          texts    = texts.map { |t| apply_prefix(t, model: model, task: task) }
 
           return generate_ollama_batch(texts: texts, model: model) if provider&.to_sym == :ollama
           return generate_azure_batch(texts: texts, model: model, dimensions: dimensions) if provider&.to_sym == :azure
@@ -75,6 +80,24 @@ module Legion
 
         private
 
+        def disabled_batch_result(texts, provider, model)
+          return nil unless provider_disabled?(provider)
+
+          model ||= resolve_model(provider)
+          texts.each_with_index.map do |_, i|
+            { vector: nil, model: model, provider: provider, dimensions: 0, index: i, error: "provider #{provider} is disabled" }
+          end
+        end
+
+        def provider_disabled?(provider)
+          return false unless provider
+
+          config = Legion::Settings.dig(:llm, :providers, provider.to_sym)
+          config.is_a?(Hash) && config[:enabled] == false
+        rescue StandardError
+          false
+        end
+
         def build_opts(model, provider, dimensions)
           target_dim = enforce_dimension? ? TARGET_DIMENSION : dimensions
           opts = { model: model }
@@ -113,7 +136,6 @@ module Legion
         def handle_embed_failure(error, text:, failed_provider:, failed_model:)
           fallback = find_fallback_provider(failed_provider)
           if fallback
-            Legion::Logging.info "Embedding failover: #{failed_provider} -> #{fallback[:provider]}" if defined?(Legion::Logging)
             generate(text: text, model: fallback[:model], provider: fallback[:provider])
           else
             { vector: nil, model: failed_model, provider: failed_provider, error: error.message }
@@ -121,37 +143,25 @@ module Legion
         end
 
         def find_fallback_provider(failed_provider)
-          chain = embedding_settings[:provider_fallback] || %w[ollama bedrock openai]
-          models = embedding_settings[:provider_models] || {}
-          started = false
+          chain = LLM.embedding_fallback_chain
+          return nil unless chain.is_a?(Array) && chain.any?
 
-          chain.each do |name|
-            sym = name.to_sym
-            if sym == failed_provider
+          started = false
+          chain.each do |entry|
+            if entry[:provider] == failed_provider&.to_sym
               started = true
               next
             end
             next unless started
+            # Skip providers that are explicitly disabled in the fallback chain
+            next if provider_disabled?(entry[:provider])
 
-            available = probe_fallback_provider(sym)
-            next unless available
-
-            model = available.is_a?(String) ? available : (models[name] || models[sym])&.to_s
-            return { provider: sym, model: model }
+            Legion::Logging.info "Embedding failover: #{failed_provider} -> #{entry[:provider]}" if defined?(Legion::Logging)
+            return entry
           end
           nil
         end
 
-        def probe_fallback_provider(sym)
-          case sym
-          when :ollama
-            LLM.send(:detect_ollama_embedding,
-                     embedding_settings[:ollama_preferred] || %w[mxbai-embed-large])
-          else
-            LLM.send(:detect_cloud_embedding, sym)
-          end
-        end
-
         def resolve_provider
           return LLM.embedding_provider if LLM.embedding_provider
 

data/lib/legion/llm/pipeline/executor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'concurrent'
+
 module Legion
   module LLM
     module Pipeline
@@ -40,6 +42,10 @@ module Legion
           response_normalization debate confidence_scoring tool_calls context_store post_response knowledge_capture response_return
         ].freeze
 
+        ASYNC_SAFE_STEPS = %i[post_response knowledge_capture response_return].freeze
+
+        ASYNC_THREAD_POOL = Concurrent::FixedThreadPool.new(4, fallback_policy: :caller_runs)
+
         def initialize(request)
           @request      = request
           @profile      = Profile.derive(request.caller)
@@ -515,12 +521,52 @@ module Legion
         end
 
         def execute_post_provider_steps
+          if async_post_enabled?
+            execute_post_provider_steps_mixed
+          else
+            POST_PROVIDER_STEPS.each do |step|
+              next if Profile.skip?(@profile, step)
+
+              execute_step(step) { send(:"step_#{step}") }
+            end
+          end
+        end
+
+        def execute_post_provider_steps_mixed
           POST_PROVIDER_STEPS.each do |step|
             next if Profile.skip?(@profile, step)
+            next if ASYNC_SAFE_STEPS.include?(step)
 
             execute_step(step) { send(:"step_#{step}") }
           end
+
+          async_steps = POST_PROVIDER_STEPS.select { |s| ASYNC_SAFE_STEPS.include?(s) }
+          return if async_steps.empty?
+
+          # Snapshot timeline and warnings before firing the async thread so that
+          # build_response (called on the main thread immediately after) reads a
+          # consistent, immutable view rather than racing with async writes.
+          @_response_timeline_snapshot  = @timeline.events.dup.freeze
+          @_response_warnings_snapshot  = @warnings.dup.freeze
+          @_response_participants_snapshot = @timeline.participants.dup.freeze
+
+          profile = @profile
+          ASYNC_THREAD_POOL.post do
+            async_steps.each do |step|
+              next if Profile.skip?(profile, step)
+
+              send(:"step_#{step}")
+            end
+          rescue StandardError => e
+            Legion::Logging.warn("[pipeline] async post-step error: #{e.message}") if defined?(Legion::Logging)
+          end
+        end
+        private :execute_post_provider_steps_mixed
+
+        def async_post_enabled?
+          Legion::LLM.settings[:pipeline_async_post_steps] == true
         end
+        private :async_post_enabled?
 
         def step_provider_call_stream(&)
           providers_tried = []
@@ -713,6 +759,12 @@ module Legion
 
           @timestamps[:returned] = Time.now
 
+          # Use pre-built snapshots when async post-steps are running concurrently
+          # to avoid reading partially-mutated timeline/warnings state.
+          timeline_events   = @_response_timeline_snapshot || @timeline.events
+          timeline_parts    = @_response_participants_snapshot || @timeline.participants
+          warnings_snapshot = @_response_warnings_snapshot || @warnings
+
           Response.build(
             request_id:      @request.id,
             conversation_id: @request.conversation_id || "conv_#{SecureRandom.hex(8)}",
@@ -723,9 +775,9 @@ module Legion
             timestamps:      @timestamps,
             enrichments:     @enrichments,
             audit:           @audit,
-            timeline:        @timeline.events,
-            participants:    @timeline.participants,
-            warnings:        @warnings,
+            timeline:        timeline_events,
+            participants:    timeline_parts,
+            warnings:        warnings_snapshot,
             tracing:         @tracing,
             caller:          @request.caller,
             classification:  @request.classification,

data/lib/legion/llm/settings.rb

@@ -6,29 +6,30 @@ module Legion
       def self.default
         model_override = ENV.fetch('ANTHROPIC_MODEL', nil)
         {
-          enabled:          true,
-          connected:        false,
-          pipeline_enabled: true,
-          default_model:    model_override,
-          default_provider: nil,
-          providers:        providers,
-          routing:          routing_defaults,
-          budget:           budget_defaults,
-          confidence:       confidence_defaults,
-          discovery:        discovery_defaults,
-          gateway:          gateway_defaults,
-          daemon:           daemon_defaults,
-          prompt_caching:   prompt_caching_defaults,
-          arbitrage:        arbitrage_defaults,
-          batch:            batch_defaults,
-          scheduling:       scheduling_defaults,
-          rag:              rag_defaults,
-          embedding:        embedding_defaults,
-          conversation:     conversation_defaults,
-          telemetry:        telemetry_defaults,
-          context_curation: context_curation_defaults,
-          debate:           debate_defaults,
-          provider_layer:   provider_layer_defaults
+          enabled:                   true,
+          connected:                 false,
+          pipeline_enabled:          true,
+          pipeline_async_post_steps: true,
+          default_model:             model_override,
+          default_provider:          nil,
+          providers:                 providers,
+          routing:                   routing_defaults,
+          budget:                    budget_defaults,
+          confidence:                confidence_defaults,
+          discovery:                 discovery_defaults,
+          gateway:                   gateway_defaults,
+          daemon:                    daemon_defaults,
+          prompt_caching:            prompt_caching_defaults,
+          arbitrage:                 arbitrage_defaults,
+          batch:                     batch_defaults,
+          scheduling:                scheduling_defaults,
+          rag:                       rag_defaults,
+          embedding:                 embedding_defaults,
+          conversation:              conversation_defaults,
+          telemetry:                 telemetry_defaults,
+          context_curation:          context_curation_defaults,
+          debate:                    debate_defaults,
+          provider_layer:            provider_layer_defaults
         }
       end
 
@@ -165,7 +166,7 @@ module Legion
         {
           dimension:         1024,
           enforce_dimension: true,
-          provider_fallback: %w[azure ollama bedrock openai],
+          provider_fallback: %w[ollama bedrock openai],
           provider_models:   {
             ollama:  'mxbai-embed-large',
             azure:   'text-embedding-3-small',

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.6.7'
+    VERSION = '0.6.8'
   end
 end

data/lib/legion/llm.rb

@@ -75,6 +75,7 @@ module Legion
         @can_embed = nil
         @embedding_provider = nil
         @embedding_model = nil
+        @embedding_fallback_chain = nil
         ProviderRegistry.reset!
         Legion::Logging.info 'Legion::LLM shut down'
       end
@@ -87,7 +88,7 @@ module Legion
         @can_embed == true
       end
 
-      attr_reader :embedding_provider, :embedding_model
+      attr_reader :embedding_provider, :embedding_model, :embedding_fallback_chain
 
       def settings
         if Legion.const_defined?('Settings', false)
@@ -630,13 +631,16 @@ module Legion
           @can_embed = true
           @embedding_provider = found[:provider]
           @embedding_model = found[:model]
+          @embedding_fallback_chain = build_embedding_fallback_chain(embedding_settings)
           Legion::Logging.info "Embedding available: #{@embedding_provider}:#{@embedding_model}"
         else
           @can_embed = false
+          @embedding_fallback_chain = []
           Legion::Logging.info 'No embedding provider available'
         end
       rescue StandardError => e
         @can_embed = false
+        @embedding_fallback_chain = []
         Legion::Logging.warn "Embedding detection failed: #{e.message}" if defined?(Legion::Logging)
       end
 
@@ -705,6 +709,28 @@ module Legion
         nil
       end
 
+      def build_embedding_fallback_chain(embedding_settings)
+        fallback = embedding_settings[:provider_fallback] || %w[ollama bedrock openai]
+        provider_models = embedding_settings[:provider_models] || {}
+        ollama_preferred = embedding_settings[:ollama_preferred] || %w[mxbai-embed-large bge-large snowflake-arctic-embed]
+
+        fallback.filter_map do |provider_name|
+          provider = provider_name.to_sym
+          next unless provider_enabled?(provider)
+
+          available = probe_embedding_provider(provider, ollama_preferred)
+          next unless available
+
+          model = available.is_a?(String) ? available : (provider_models[provider_name] || provider_models[provider])&.to_s
+          { provider: provider, model: model }
+        end
+      end
+
+      def provider_enabled?(provider)
+        config = settings.dig(:providers, provider)
+        config.is_a?(Hash) && config[:enabled] != false
+      end
+
       def run_discovery
         return unless settings.dig(:providers, :ollama, :enabled)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.6.7
+  version: 0.6.8
 platform: ruby
 authors:
 - Esity