legion-data 1.10.7 → 1.10.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc598e354968c28944d6e897feb5853d57987fed49eaf0cd16f319abcfbd8cf8
-  data.tar.gz: ebc17c4d4b8c49585bcdb0a49803c4988e8d55970ddde56b294746d810193f63
+  metadata.gz: 8d929f0ed083e64145e6022ee40af61b677b852513e071ec9a986267837953a2
+  data.tar.gz: ed00e7cb0c7670378212cc1f8df39c893f0efbaf7a71961188a411a1956dcab9
 SHA512:
-  metadata.gz: 7f36631ecd03c493fa2dfd3b347a4f16765b7900545d31306e4b73f0418a455a99c15ab55a8de83ad86c123befede09a222a67a6095f624923028c585ca9c7d0
-  data.tar.gz: 9a1783c655df6dce9c23a4a878645cf196e4463d0763d95c380029263a8deeccce7cfce8e66ede4843a04c68e8369232d8d4b785460bd70181d4ad1adcf1c01b
+  metadata.gz: fdf3cf290669b970b523c3de8e5d6b42b9b14d877bd7260155c2067a708745cd59120605059dc1d5af85fd94f0a63d0c1e4e39ae4b7870409357572085b1d5f2
+  data.tar.gz: 3b1c0bb52ba3bfcc0249f6a6a4c0185b213f7e6cd8201989f5cd0279be1d9e1df1e364c0479d48920397f7e693ff5a63f557048acdaaefecdd32651e212922c7

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Legion::Data Changelog
 
+## [1.10.9] - 2026-07-31
+### Fixed
+- **Postgres session timeouts**: Apply `statement_timeout` (default 5000ms) and `lock_timeout` (default 2000ms) via `after_connect` hook on every new Postgres connection. Prevents unbounded `SELECT` or lock-wait from hanging the process indefinitely — the root cause of ~1h50m connection wedges.
+- **TCP keepalives on Postgres**: Wire libpq `keepalives`, `keepalives_idle` (10s), `keepalives_interval` (5s), `keepalives_count` (3), and `tcp_user_timeout` (15000ms) through the Sequel connection hash. Detects half-open TCP sockets within ~25s instead of relying on OS defaults (often 2+ hours).
+- **Force-disconnect on shutdown**: `shutdown` now waits up to `shutdown_timeout` (default 5s) for checked-out connections to return, then force-disconnects all remaining connections. Previously, `disconnect` only closed idle pool connections — a connection stuck in `select()` survived forever, hanging process exit.
+- **Adapter timeout defaults tightened**: Postgres `connect_timeout` reduced from 20s to 5s; MySQL `connect_timeout` reduced from 120s to 5s, `read_timeout` and `write_timeout` default to 5s. The old 20s/120s values allowed connection attempts to block worker threads far too long under network partition.
+- **MySQL read/write timeout wiring preserved**: `read_timeout` and `write_timeout` remain in `ADAPTER_KEYS[:mysql2]` where Sequel's mysql2 adapter honors them. For Postgres, the equivalent is `statement_timeout` (query bound) plus TCP keepalives (socket bound) — `read_timeout` is not a libpq/Sequel-postgres concept, so it is correctly absent from the Postgres key list.
+
+## [1.10.8] - 2026-07-23
+### Fixed
+- Detect auth failures (`role does not exist`, `password authentication failed`) on Sequel pool connections and trigger immediate Vault lease reissue via LeaseManager instead of retrying dead credentials forever. 30-second cooldown prevents reissue storms during bulk credential rotation.
+
 ## [1.10.7] - 2026-07-15
 ### Fixed
 - Fail loud with actionable error when unresolved lease:// credentials reach connection

data/lib/legion/data/auth_failure_handler.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'legion/logging/helper'
+
+module Legion
+  module Data
+    module AuthFailureHandler
+      extend Legion::Logging::Helper
+
+      AUTH_FAILURE_PATTERNS = [
+        /role .* does not exist/i,
+        /password authentication failed/i,
+        /authentication failed/i,
+        /no pg_hba\.conf entry/i,
+        /permission denied for table/i,
+        /permission denied for relation/i
+      ].freeze
+
+      REISSUE_COOLDOWN = 30
+
+      @last_reissue_at = nil
+      @mutex = Mutex.new
+
+      module SequelHook
+        def connect(server)
+          super
+        rescue StandardError => e
+          Legion::Data::AuthFailureHandler.handle(e)
+          raise
+        end
+
+        def raise_error(exception, opts = Sequel::OPTS)
+          Legion::Data::AuthFailureHandler.handle(exception)
+          super
+        end
+      end
+
+      class << self
+        def install(sequel_db)
+          sequel_db.singleton_class.prepend(SequelHook)
+        end
+
+        def handle(error)
+          return unless auth_failure?(error)
+          return if on_cooldown?
+
+          request_reissue(error)
+        end
+
+        def auth_failure?(error)
+          message = error.message.to_s
+          AUTH_FAILURE_PATTERNS.any? { |pattern| message.match?(pattern) }
+        end
+
+        def on_cooldown?
+          @mutex.synchronize do
+            return false unless @last_reissue_at
+
+            (Time.now - @last_reissue_at) < REISSUE_COOLDOWN
+          end
+        end
+
+        def request_reissue(error, adapter: :postgresql)
+          @mutex.synchronize { @last_reissue_at = Time.now }
+          log.error("Legion::Data auth failure detected: #{error.message} — requesting lease reissue")
+
+          return unless defined?(Legion::Crypt::LeaseManager)
+
+          Legion::Crypt::LeaseManager.instance.reissue_lease(adapter)
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: true, operation: :auth_failure_reissue)
+        end
+
+        def reset!
+          @mutex.synchronize { @last_reissue_at = nil }
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/connection.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/logging/helper'
+require 'legion/data/auth_failure_handler'
 
 require 'fileutils'
 require 'sequel'
@@ -18,14 +19,18 @@ module Legion
 
       ADAPTER_KEYS = {
         sqlite:   %i[timeout readonly disable_dqs],
-        postgres: %i[connect_timeout sslmode sslrootcert search_path],
+        postgres: %i[connect_timeout sslmode sslrootcert search_path
+                     keepalives keepalives_idle keepalives_interval keepalives_count
+                     tcp_user_timeout],
         mysql2:   %i[connect_timeout read_timeout write_timeout encoding sql_mode]
       }.freeze
 
       ADAPTER_DEFAULTS = {
         sqlite:   { timeout: 5000, readonly: false, disable_dqs: true },
-        postgres: { connect_timeout: 20, sslmode: 'disable' },
-        mysql2:   { connect_timeout: 120, encoding: 'utf8mb4' }
+        postgres: { connect_timeout: 5, sslmode: 'disable',
+                    keepalives: 1, keepalives_idle: 10, keepalives_interval: 5,
+                    keepalives_count: 3, tcp_user_timeout: 15_000 },
+        mysql2:   { connect_timeout: 5, read_timeout: 5, write_timeout: 5, encoding: 'utf8mb4' }
       }.freeze
 
       QUERY_LOG_DIR = File.expand_path('~/.legionio/logs').freeze
@@ -271,7 +276,10 @@ module Legion
         end
 
         def shutdown
-          @sequel&.disconnect
+          if @sequel
+            timeout = Legion::Settings[:data][:shutdown_timeout]
+            force_disconnect_pool(timeout: timeout)
+          end
           @query_file_logger&.close
           @query_file_logger = nil
           @fallback_active = false
@@ -299,6 +307,7 @@ module Legion
           @sequel.opts[:password] = new_pass
 
           @sequel.disconnect
+          expire_all_pooled_connections
 
           @sequel.test_connection
           log.info("reconnect_with_fresh_creds: rotated credentials (#{old_user} → #{new_user})")
@@ -309,6 +318,16 @@ module Legion
           false
         end
 
+        def expire_all_pooled_connections
+          pool = @sequel.pool
+          return unless pool.instance_variable_defined?(:@connection_expiration_timestamps)
+
+          timestamps = pool.instance_variable_get(:@connection_expiration_timestamps)
+          timestamps.each_key { |conn| timestamps[conn] = [0, 0].freeze }
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :expire_all_pooled_connections)
+        end
+
         def connect_with_replicas
           return unless adapter == :postgres
 
@@ -397,15 +416,49 @@ module Legion
             conn_host = actual[:host] || '127.0.0.1'
             conn_port = actual[:port]
             conn_db   = actual[:database] || actual[:db]
-            log.info "Connected to #{adapter}://#{conn_user}@#{conn_host}:#{conn_port}/#{conn_db}"
+            lease_id  = current_lease_id
+            msg = "Connected to #{adapter}://#{conn_user}@#{conn_host}:#{conn_port}/#{conn_db}"
+            msg += " lease_id=#{lease_id}" if lease_id
+            log.info msg
           end
         end
 
+        def current_lease_id
+          return unless defined?(Legion::Crypt::LeaseManager)
+
+          Legion::Crypt::LeaseManager.instance.active_leases.dig(:postgresql, :lease_id)
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :current_lease_id)
+          nil
+        end
+
         def dev_fallback?
           data_settings = Legion::Settings[:data]
           data_settings[:dev_mode] == true && data_settings[:dev_fallback] != false
         end
 
+        def force_disconnect_pool(timeout:)
+          pool = @sequel.pool
+          in_use = pool.size - (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
+
+          if in_use.positive?
+            log.warn("Legion::Data shutdown: #{in_use} connection(s) still checked out, waiting up to #{timeout}s")
+            deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
+            while ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) < deadline
+              break if pool.size <= (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
+
+              sleep 0.1
+            end
+            remaining = pool.size - (pool.respond_to?(:available_connections) ? pool.available_connections.size : 0)
+            log.warn("Legion::Data shutdown: force-disconnecting #{remaining} wedged connection(s)") if remaining.positive?
+          end
+
+          @sequel.disconnect
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :force_disconnect_pool)
+          @sequel&.disconnect rescue nil # rubocop:disable Style/RescueModifier
+        end
+
         def sqlite_path
           path = Legion::Settings[:data][:creds][:database] || 'legionio.db'
           return path if File.absolute_path?(path)
@@ -489,6 +542,11 @@ module Legion
           tuning[:connection_expiration]          = data[:connection_expiration]
           tuning[:connection_expiration_timeout]  = data[:connection_expiration_timeout]
 
+          # Session timeouts (postgres)
+          tuning[:statement_timeout]   = data[:statement_timeout]
+          tuning[:lock_timeout]        = data[:lock_timeout]
+          tuning[:shutdown_timeout]    = data[:shutdown_timeout]
+
           # Adapter-specific (only keys relevant to current adapter)
           defaults = ADAPTER_DEFAULTS.fetch(adapter, {})
           ADAPTER_KEYS.fetch(adapter, []).each do |key|
@@ -597,6 +655,7 @@ module Legion
           if adapter == :postgres
             Sequel.extension(:pg_array)
             @sequel.extension(:pg_array)
+            install_postgres_session_timeouts
           end
 
           if data[:connection_validation] != false
@@ -608,10 +667,33 @@ module Legion
             @sequel.extension(:connection_expiration)
             @sequel.pool.connection_expiration_timeout = data[:connection_expiration_timeout]
           end
+
+          install_auth_failure_hook
         rescue StandardError => e
           handle_exception(e, level: :warn, handled: true, operation: :configure_extensions, adapter: adapter)
         end
 
+        def install_postgres_session_timeouts
+          data = Legion::Settings[:data]
+          stmt_timeout = data[:statement_timeout]
+          lck_timeout = data[:lock_timeout]
+
+          @sequel.pool.after_connect = proc do |conn|
+            conn.exec("SET statement_timeout = '#{stmt_timeout.to_i}ms'") if stmt_timeout
+            conn.exec("SET lock_timeout = '#{lck_timeout.to_i}ms'") if lck_timeout
+          end
+
+          log.info("Postgres session timeouts: statement_timeout=#{stmt_timeout}ms, lock_timeout=#{lck_timeout}ms")
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :install_postgres_session_timeouts)
+        end
+
+        def install_auth_failure_hook
+          Legion::Data::AuthFailureHandler.install(@sequel)
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :install_auth_failure_hook)
+        end
+
         def build_data_logger
           tagged = if defined?(Legion::Logging::TaggedLogger) && respond_to?(:tagged_logger_settings, true)
                      Legion::Logging::TaggedLogger.new(

data/lib/legion/data/settings.rb

@@ -58,6 +58,28 @@ module Legion
           connection_expiration:         true,
           connection_expiration_timeout: 14_400,
 
+          # Postgres session timeouts (milliseconds, applied via SET on each new connection).
+          # statement_timeout bounds any single query; lock_timeout bounds lock acquisition.
+          # These prevent a wedged SELECT or lock wait from hanging the process indefinitely.
+          statement_timeout:             5000,
+          lock_timeout:                  2000,
+
+          # TCP keepalives for Postgres (libpq connection params).
+          # Detect half-open sockets where the peer vanished without FIN/RST.
+          # keepalives_idle: seconds before first keepalive probe after idle
+          # keepalives_interval: seconds between probes
+          # keepalives_count: failed probes before declaring connection dead
+          # tcp_user_timeout: total ms for unacked data before kernel drops connection (Linux)
+          keepalives:                    1,
+          keepalives_idle:               10,
+          keepalives_interval:           5,
+          keepalives_count:              3,
+          tcp_user_timeout:              15_000,
+
+          # Shutdown: max seconds to wait for checked-out connections to return before
+          # force-disconnecting them.
+          shutdown_timeout:              5,
+
           # Adapter-specific (nil = use adapter built-in default)
           connect_timeout:               nil,
           read_timeout:                  nil,

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.10.7'
+    VERSION = '1.10.9'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.10.7
+  version: 1.10.9
 platform: ruby
 authors:
 - Esity
@@ -124,6 +124,7 @@ files:
 - lib/legion/data/archiver.rb
 - lib/legion/data/audit_log_hash_chain.rb
 - lib/legion/data/audit_record.rb
+- lib/legion/data/auth_failure_handler.rb
 - lib/legion/data/connection.rb
 - lib/legion/data/encryption/cipher.rb
 - lib/legion/data/encryption/key_provider.rb