legionio 1.5.13 → 1.5.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4815cb38dfcd2dc20af4e57d1ec14f63a9bf7876477b70fc11696b775685addf
-  data.tar.gz: 79f424e95bbe364be35ef6a1b5b11da3ef476dda096443a303b0d8136ffc79e5
+  metadata.gz: 213403cc9241316f3d19263526f688166a256ecb38dd2fb45b4f782c9067e9e0
+  data.tar.gz: e93b0000542370d69c6b46323e217d3cace04429cf061dd2ba748040a560175a
 SHA512:
-  metadata.gz: 4edcd4fabaa737d35e6176668dc63cd6091533240f0ca038250b2b44652130590ad04ee144ed58df2eca1aa3d5313353b46a99069b42eefd2d224384694209a5
-  data.tar.gz: 16d72dd0db3df555b27defbe22fe1a3b27e2aa11d11674f57d89bc8de593f624aeb5c5ae6fcb589a5f085b0eea9619b0fd6ef56349cf749a3819210f5cc7b7ca
+  metadata.gz: c272f3d6e719737682c354a8313d5f3b93797ac44da5fea75f7e0473ffd27a6075eb2ba67665f2e6dfa55e37f3c2c986ae66ae3dcc4cdfbaa78c0fb2517a7fe5
+  data.tar.gz: fb92bae4cf4fddf1a9a47b88eb611d13bb4dfe7162315a560a9c9b75fd5250f41cf30a80cef166e88488794ef16a7c0d462e6018de4e460409e89d35ad33f41a

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Legion Changelog
 
+## [1.5.14] - 2026-03-25
+
+### Fixed
+- Shutdown no longer hangs when network is unreachable — all component shutdowns wrapped in bounded timeouts via `shutdown_component` helper (#30)
+- Reload path also wrapped with same timeout guards to prevent hangs during network-triggered reload (#30)
+
+### Added
+- Network watchdog: background `Concurrent::TimerTask` monitors transport/data/cache connectivity, pauses actors after sustained failures, triggers `Legion.reload` when network restores (#30)
+- `Legion::Extensions.pause_actors` suspends all `Every` timer tasks without destroying instances (#30)
+- Watchdog is feature-flagged via `network.watchdog.enabled` (default: false), configurable threshold and interval (#30)
+
 ## [1.5.13] - 2026-03-25
 
 ### Fixed

data/lib/legion/extensions.rb

@@ -96,6 +96,16 @@ module Legion
         Legion::Logging.info "Successfully shut down all actors (#{(Time.now - shutdown_start).round(1)}s)"
       end
 
+      def pause_actors
+        @running_instances&.each do |inst|
+          timer = inst.instance_variable_get(:@timer)
+          timer&.shutdown if timer.respond_to?(:shutdown)
+        rescue StandardError => e
+          Legion::Logging.error "pause_actors: #{e.class}: #{e.message}" if defined?(Legion::Logging)
+        end
+        Legion::Logging.warn 'All actors paused' if defined?(Legion::Logging)
+      end
+
       def load_extensions
         @extensions ||= []
         @loaded_extensions ||= []

data/lib/legion/service.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'timeout'
 require_relative 'readiness'
 require_relative 'process_role'
 
@@ -130,6 +131,7 @@ module Legion
       api_settings = Legion::Settings[:api] || {}
       @api_enabled = api && api_settings.fetch(:enabled, true)
       setup_api if @api_enabled
+      setup_network_watchdog
       Legion::Settings[:client][:ready] = true
       Legion::Events.emit('service.ready')
     end
@@ -465,13 +467,14 @@ module Legion
       Legion::Settings[:client][:shutting_down] = true
       Legion::Events.emit('service.shutting_down')
 
+      shutdown_network_watchdog
       shutdown_audit_archiver
       shutdown_api
 
       Legion::Metrics.reset! if defined?(Legion::Metrics)
 
       if defined?(Legion::Gaia) && Legion::Gaia.respond_to?(:started?) && Legion::Gaia.started?
-        Legion::Gaia.shutdown
+        shutdown_component('Gaia') { Legion::Gaia.shutdown }
         Legion::Readiness.mark_not_ready(:gaia)
       end
 
@@ -480,32 +483,33 @@ module Legion
         @cluster_leader = nil
       end
 
-      Legion::Extensions.shutdown
+      ext_timeout = Legion::Settings.dig(:extensions, :shutdown_timeout) || 15
+      shutdown_component('Extensions', timeout: ext_timeout) { Legion::Extensions.shutdown }
       Legion::Readiness.mark_not_ready(:extensions)
 
       if Legion::Settings[:llm]&.dig(:connected)
-        Legion::LLM.shutdown
+        shutdown_component('LLM') { Legion::LLM.shutdown }
         Legion::Readiness.mark_not_ready(:llm)
       end
 
       if defined?(Legion::Rbac) && Legion::Settings[:rbac]&.dig(:connected)
-        Legion::Rbac.shutdown
+        shutdown_component('Rbac') { Legion::Rbac.shutdown }
         Legion::Readiness.mark_not_ready(:rbac)
       end
 
-      Legion::Data.shutdown if Legion::Settings[:data][:connected]
+      shutdown_component('Data') { Legion::Data.shutdown } if Legion::Settings[:data][:connected]
       Legion::Readiness.mark_not_ready(:data)
 
       Legion::Leader.reset! if defined?(Legion::Leader)
 
-      Legion::Cache.shutdown
+      shutdown_component('Cache') { Legion::Cache.shutdown }
       Legion::Readiness.mark_not_ready(:cache)
 
-      Legion::Transport::Connection.shutdown
+      shutdown_component('Transport') { Legion::Transport::Connection.shutdown }
       Legion::Readiness.mark_not_ready(:transport)
 
       shutdown_mtls_rotation
-      Legion::Crypt.shutdown
+      shutdown_component('Crypt') { Legion::Crypt.shutdown }
       Legion::Readiness.mark_not_ready(:crypt)
 
       Legion::Settings[:client][:ready] = false
@@ -513,29 +517,34 @@ module Legion
     end
 
     def reload
+      return if @reloading
+
+      @reloading = true
       Legion::Logging.info 'Legion::Service.reload was called'
       Legion::Settings[:client][:ready] = false
 
+      shutdown_network_watchdog
       shutdown_api
 
       if defined?(Legion::Gaia) && Legion::Gaia.respond_to?(:started?) && Legion::Gaia.started?
-        Legion::Gaia.shutdown
+        shutdown_component('Gaia') { Legion::Gaia.shutdown }
         Legion::Readiness.mark_not_ready(:gaia)
       end
 
-      Legion::Extensions.shutdown
+      ext_timeout = Legion::Settings.dig(:extensions, :shutdown_timeout) || 15
+      shutdown_component('Extensions', timeout: ext_timeout) { Legion::Extensions.shutdown }
       Legion::Readiness.mark_not_ready(:extensions)
 
-      Legion::Data.shutdown
+      shutdown_component('Data') { Legion::Data.shutdown }
       Legion::Readiness.mark_not_ready(:data)
 
-      Legion::Cache.shutdown
+      shutdown_component('Cache') { Legion::Cache.shutdown }
       Legion::Readiness.mark_not_ready(:cache)
 
-      Legion::Transport::Connection.shutdown
+      shutdown_component('Transport') { Legion::Transport::Connection.shutdown }
       Legion::Readiness.mark_not_ready(:transport)
 
-      Legion::Crypt.shutdown
+      shutdown_component('Crypt') { Legion::Crypt.shutdown }
       Legion::Readiness.mark_not_ready(:crypt)
 
       Legion::Readiness.wait_until_not_ready(:transport, :data, :cache, :crypt)
@@ -569,9 +578,12 @@ module Legion
 
       Legion::Crypt.cs
       setup_api if @api_enabled
+      setup_network_watchdog
       Legion::Settings[:client][:ready] = true
       Legion::Events.emit('service.ready')
       Legion::Logging.info 'Legion has been reloaded'
+    ensure
+      @reloading = false
     end
 
     def load_extensions
@@ -630,6 +642,67 @@ module Legion
       nil
     end
 
+    def shutdown_component(name, timeout: 5, &)
+      Timeout.timeout(timeout, &)
+    rescue Timeout::Error
+      Legion::Logging.warn "#{name} shutdown timed out after #{timeout}s, forcing"
+    rescue StandardError => e
+      Legion::Logging.warn "#{name} shutdown error: #{e.class}: #{e.message}"
+    end
+
+    def setup_network_watchdog
+      return unless Legion::Settings.dig(:network, :watchdog, :enabled)
+
+      @consecutive_failures = Concurrent::AtomicFixnum.new(0)
+      threshold = Legion::Settings.dig(:network, :watchdog, :failure_threshold) || 5
+      interval = Legion::Settings.dig(:network, :watchdog, :check_interval) || 15
+
+      @network_watchdog = Concurrent::TimerTask.new(execution_interval: interval) do
+        if network_healthy?
+          prev = @consecutive_failures.value
+          @consecutive_failures.value = 0
+          if prev >= threshold
+            Legion::Logging.info '[Watchdog] Network restored, triggering reload'
+            Thread.new { Legion.reload } unless @reloading
+          end
+        else
+          count = @consecutive_failures.increment
+          Legion::Logging.warn "[Watchdog] Network check failed (#{count}/#{threshold})"
+          if count == threshold
+            Legion::Logging.error '[Watchdog] Network failure threshold reached, pausing actors'
+            Legion::Extensions.pause_actors if Legion::Extensions.respond_to?(:pause_actors)
+          end
+        end
+      rescue StandardError => e
+        Legion::Logging.debug "[Watchdog] check error: #{e.message}"
+      end
+      @network_watchdog.execute
+      Legion::Logging.info "[Watchdog] Network watchdog started (interval=#{interval}s, threshold=#{threshold})"
+    rescue StandardError => e
+      Legion::Logging.warn "Network watchdog setup failed: #{e.message}"
+    end
+
+    def shutdown_network_watchdog
+      @network_watchdog&.shutdown
+      @network_watchdog = nil
+    end
+
+    def network_healthy?
+      return true if defined?(Legion::Transport::Connection) && Legion::Transport::Connection.lite_mode?
+
+      checks = []
+      checks << Legion::Transport::Connection.session_open? if Legion::Settings[:transport][:connected]
+      if Legion::Settings[:data][:connected] && defined?(Legion::Data::Connection)
+        checks << (Legion::Data::Connection.sequel&.test_connection rescue false) # rubocop:disable Style/RescueModifier
+      end
+      checks << Legion::Cache.connected? if Legion::Settings[:cache][:connected] && defined?(Legion::Cache)
+      return true if checks.empty?
+
+      checks.any?
+    rescue StandardError
+      false
+    end
+
     private
 
     def port_in_use?(bind, port)

data/lib/legion/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legion
-  VERSION = '1.5.13'
+  VERSION = '1.5.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legionio
 version: !ruby/object:Gem::Version
-  version: 1.5.13
+  version: 1.5.14
 platform: ruby
 authors:
 - Esity