datadog 2.12.1 → 2.19.0

data/lib/datadog/core/configuration/option.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'stable_config'
 require_relative '../utils/safe_dup'
 
 module Datadog
@@ -21,7 +22,7 @@ module Datadog
           # Represents an Option precedence level.
           # Each precedence has a `numeric` value; higher values means higher precedence.
           # `name` is for inspection purposes only.
-          Value = Struct.new(:numeric, :name) do
+          Value = Struct.new(:numeric, :name, :origin) do
             include Comparable
 
             def <=>(other)
@@ -32,17 +33,25 @@ module Datadog
           end
 
           # Remote configuration provided through the Datadog app.
-          REMOTE_CONFIGURATION = Value.new(2, :remote_configuration).freeze
+          REMOTE_CONFIGURATION = Value.new(5, :remote_configuration, 'remote_config').freeze
 
           # Configuration provided in Ruby code, in this same process
-          # or via Environment variable
-          PROGRAMMATIC = Value.new(1, :programmatic).freeze
+          PROGRAMMATIC = Value.new(4, :programmatic, 'code').freeze
+
+          # Configuration provided by fleet managed stable config
+          FLEET_STABLE = Value.new(3, :fleet_stable, 'fleet_stable_config').freeze
+
+          # Configuration provided via environment variable
+          ENVIRONMENT = Value.new(2, :environment, 'env_var').freeze
+
+          # Configuration provided by local stable config file
+          LOCAL_STABLE = Value.new(1, :local_stable, 'local_stable_config').freeze
 
           # Configuration that comes from default values
-          DEFAULT = Value.new(0, :default).freeze
+          DEFAULT = Value.new(0, :default, 'default').freeze
 
           # All precedences, sorted from highest to lowest
-          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, DEFAULT].sort.reverse.freeze
+          LIST = [REMOTE_CONFIGURATION, PROGRAMMATIC, FLEET_STABLE, ENVIRONMENT, LOCAL_STABLE, DEFAULT].sort.reverse.freeze
         end
 
         def initialize(definition, context)
@@ -118,23 +127,29 @@ module Datadog
         end
 
         def get
-          if @is_set
-            @value
-          else
-            set_value_from_env_or_default
+          unless @is_set
+            # Ensures that both the default value and the environment value are set.
+            # This approach handles scenarios where an environment value is unset
+            # by falling back to the default value consistently.
+            set_default_value
+            set_customer_stable_config_value
+            set_env_value
+            set_fleet_stable_config_value
           end
+
+          @value
         end
 
         def reset
           @value = if definition.resetter
-                     # Don't change @is_set to false; custom resetters are
-                     # responsible for changing @value back to a good state.
-                     # Setting @is_set = false would cause a default to be applied.
-                     context_exec(@value, &definition.resetter)
-                   else
-                     @is_set = false
-                     nil
-                   end
+            # Don't change @is_set to false; custom resetters are
+            # responsible for changing @value back to a good state.
+            # Setting @is_set = false would cause a default to be applied.
+            context_exec(@value, &definition.resetter)
+          else
+            @is_set = false
+            nil
+          end
 
           # Reset back to the lowest precedence, to allow all `set`s to succeed right after a reset.
           @precedence_set = Precedence::DEFAULT
@@ -192,7 +207,7 @@ module Datadog
           when :bool
             string_value = value.strip
             string_value = string_value.downcase
-            string_value == 'true' || string_value == '1' # rubocop:disable Style/MultipleComparison
+            string_value == 'true' || string_value == '1'
           when :string, NilClass
             value
           else
@@ -212,20 +227,20 @@ module Datadog
 
           unless valid_type
             raise_error = if @definition.type_options[:nilable]
-                            !value.is_a?(NilClass)
-                          else
-                            true
-                          end
+              !value.is_a?(NilClass)
+            else
+              true
+            end
           end
 
           if raise_error
             error_msg = if @definition.type_options[:nilable]
-                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
-                          "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})."\
-                        else
-                          "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a "\
-                          "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})."\
-                        end
+              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "#{@definition.type} or `nil`, but a `#{value.class}` was provided (#{value.inspect})." \
+            else
+              "The setting `#{@definition.name}` inside your app's `Datadog.configure` block expects a " \
+              "#{@definition.type}, but a `#{value.class}` was provided (#{value.inspect})." \
+            end
 
             error_msg = "#{error_msg} Please update your `configure` block. "
 
@@ -256,7 +271,8 @@ module Datadog
           when NilClass
             true # No validation is performed when option is typeless
           else
-            raise ArgumentError, "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
+            raise InvalidDefinitionError,
+              "The option #{@definition.name} is using an unsupported type option `#{@definition.type}`"
           end
         end
 
@@ -269,7 +285,7 @@ module Datadog
             @resolved_env = resolved_env
             # Store original value to ensure we can always safely call `#internal_set`
             # when restoring a value from `@value_per_precedence`, and we are only running `definition.setter`
-            # on the original value, not on a valud that has already been processed by `definition.setter`.
+            # on the original value, not on a value that has already been processed by `definition.setter`.
             @value_per_precedence[precedence] = value
             context_exec(v, old_value, precedence, &definition.after_set) if definition.after_set
           end
@@ -283,39 +299,59 @@ module Datadog
           @context.instance_eval(&block)
         end
 
-        def set_value_from_env_or_default
+        def set_default_value
+          set(default_value, precedence: Precedence::DEFAULT)
+        end
+
+        def set_env_value
+          value, resolved_env = get_value_and_resolved_env_from(ENV)
+          set(value, precedence: Precedence::ENVIRONMENT, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_customer_stable_config_value
+          customer_config = StableConfig.configuration.dig(:local, :config)
+          return if customer_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(customer_config, source: 'local stable config')
+          set(value, precedence: Precedence::LOCAL_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def set_fleet_stable_config_value
+          fleet_config = StableConfig.configuration.dig(:fleet, :config)
+          return if fleet_config.nil?
+
+          value, resolved_env = get_value_and_resolved_env_from(fleet_config, source: 'fleet stable config')
+          set(value, precedence: Precedence::FLEET_STABLE, resolved_env: resolved_env) unless value.nil?
+        end
+
+        def get_value_and_resolved_env_from(env_vars, source: 'environment variable')
           value = nil
-          precedence = nil
           resolved_env = nil
 
           if definition.env
             Array(definition.env).each do |env|
-              next if ENV[env].nil?
+              next if env_vars[env].nil?
 
               resolved_env = env
-              value = coerce_env_variable(ENV[env])
-              precedence = Precedence::PROGRAMMATIC
+              value = coerce_env_variable(env_vars[env])
               break
             end
           end
 
-          if value.nil? && definition.deprecated_env && ENV[definition.deprecated_env]
+          if value.nil? && definition.deprecated_env && env_vars[definition.deprecated_env]
             resolved_env = definition.deprecated_env
-            value = coerce_env_variable(ENV[definition.deprecated_env])
-            precedence = Precedence::PROGRAMMATIC
+            value = coerce_env_variable(env_vars[definition.deprecated_env])
 
             Datadog::Core.log_deprecation do
-              "#{definition.deprecated_env} environment variable is deprecated, use #{definition.env} instead."
+              "#{definition.deprecated_env} #{source} is deprecated, use #{definition.env} instead."
             end
           end
 
-          option_value = value.nil? ? default_value : value
-
-          set(option_value, precedence: precedence || Precedence::DEFAULT, resolved_env: resolved_env)
+          [value, resolved_env]
         rescue ArgumentError
           raise ArgumentError,
-            "Expected environment variable #{resolved_env} to be a #{@definition.type}, " \
-                              "but '#{ENV[resolved_env]}' was provided"
+            "Expected #{source} #{resolved_env} to be a #{@definition.type}, " \
+                              "but '#{env_vars[resolved_env]}' was provided"
         end
 
         # Anchor object that represents a value that is not set.

data/lib/datadog/core/configuration/option_definition.rb

@@ -71,11 +71,11 @@ module Datadog
             validate_options!
           end
 
-          def env(value)
+          def env(value) # standard:disable Style/TrivialAccessors
             @env = value
           end
 
-          def deprecated_env(value)
+          def deprecated_env(value) # standard:disable Style/TrivialAccessors
             @deprecated_env = value
           end
 
@@ -111,7 +111,7 @@ module Datadog
 
           def type(value, nilable: false)
             @type = value
-            @type_options = { nilable: nilable }
+            @type_options = {nilable: nilable}
 
             value
           end
@@ -156,7 +156,7 @@ module Datadog
             if !@default.nil? && @default_proc
               raise InvalidOptionError,
                 'Using `default` and `default_proc` is not allowed. Please use one or the other.' \
-                                'If you want to store a block as the default value use `default_proc`'\
+                                'If you want to store a block as the default value use `default_proc`' \
                                 ' otherwise use `default`'
             end
           end

data/lib/datadog/core/configuration/options.rb

@@ -18,7 +18,7 @@ module Datadog
         module ClassMethods
           def options
             # Allows for class inheritance of option definitions
-            @options ||= superclass <= Options ? superclass.options.dup : {}
+            @options ||= (superclass <= Options) ? superclass.options.dup : {}
           end
 
           protected
@@ -45,7 +45,7 @@ module Datadog
               option_name.to_sym => proc do
                 get_option(option_name)
               end,
-              "#{option_name}=".to_sym => proc do |value|
+              :"#{option_name}=" => proc do |value|
                 set_option(option_name, value)
               end
             }
@@ -117,7 +117,7 @@ module Datadog
           end
 
           def resolved_env(name)
-            return options[name].resolved_env if options.key?(name)
+            options[name].resolved_env if options.key?(name)
           end
 
           def assert_valid_option!(name)

data/lib/datadog/core/configuration/settings.rb

@@ -18,7 +18,7 @@ module Datadog
     module Configuration
       # Global configuration settings for the Datadog library.
       # @public_api
-      # rubocop:disable Metrics/BlockLength
+      # standard:disable Metrics/BlockLength
       class Settings
         include Base
 
@@ -123,7 +123,7 @@ module Datadog
           # @return [Boolean]
           option :debug do |o|
             o.env [Datadog::Core::Configuration::Ext::Diagnostics::ENV_DEBUG_ENABLED,
-                   Datadog::Core::Configuration::Ext::Diagnostics::ENV_OTEL_LOG_LEVEL]
+              Datadog::Core::Configuration::Ext::Diagnostics::ENV_OTEL_LOG_LEVEL]
             o.default false
             o.type :bool
             o.env_parser do |value|
@@ -137,7 +137,7 @@ module Datadog
             o.after_set do |enabled|
               # Enable rich debug print statements.
               # We do not need to unnecessarily load 'pp' unless in debugging mode.
-              require 'pp' if enabled
+              require 'pp' if enabled # standard:disable Lint/RedundantRequireStatement
             end
           end
 
@@ -313,7 +313,7 @@ module Datadog
 
             # Can be used to enable/disable the collection of heap profiles.
             #
-            # This feature is alpha and disabled by default
+            # This feature is in preview and disabled by default. Requires Ruby 3.1+.
             #
             # @warn To enable heap profiling you are required to also enable allocation profiling.
             #
@@ -326,12 +326,12 @@ module Datadog
 
             # Can be used to enable/disable the collection of heap size profiles.
             #
-            # This feature is alpha and enabled by default when heap profiling is enabled.
+            # This feature is in preview and by default is enabled whenever heap profiling is enabled.
             #
-            # @warn To enable heap size profiling you are required to also enable allocation and heap profiling.
+            # @warn Heap size profiling depends on allocation and heap profiling, so they must be enabled as well.
             #
-            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SIZE_ENABLED` environment variable as a boolean, otherwise
-            # whatever the value of DD_PROFILING_EXPERIMENTAL_HEAP_ENABLED is.
+            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SIZE_ENABLED` environment variable as a boolean, otherwise it
+            # follows the value of `experimental_heap_enabled`.
             option :experimental_heap_size_enabled do |o|
               o.type :bool
               o.env 'DD_PROFILING_EXPERIMENTAL_HEAP_SIZE_ENABLED'
@@ -341,17 +341,19 @@ module Datadog
             # Can be used to configure the heap sampling rate: a heap sample will be collected for every x allocation
             # samples.
             #
-            # The lower the value, the more accuracy in heap tracking but the bigger the overhead. In particular, a
-            # value of 1 will track ALL allocations samples for heap profiles.
+            # The higher the value, the less accuracy in heap tracking but the smaller the overhead.
+            #
+            # If you needed to tweak this, please tell us why on <https://github.com/DataDog/dd-trace-rb/issues/new>,
+            # so we can fix it!
             #
             # The effective heap sampling rate in terms of allocations (not allocation samples) can be calculated via
             # effective_heap_sample_rate = allocation_sample_rate * heap_sample_rate.
             #
-            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SAMPLE_RATE` environment variable, otherwise `10`.
+            # @default `DD_PROFILING_EXPERIMENTAL_HEAP_SAMPLE_RATE` environment variable, otherwise `1`.
             option :experimental_heap_sample_rate do |o|
               o.type :int
               o.env 'DD_PROFILING_EXPERIMENTAL_HEAP_SAMPLE_RATE'
-              o.default 10
+              o.default 1
             end
 
             # Can be used to disable checking which version of `libmysqlclient` is being used by the `mysql2` gem.
@@ -454,7 +456,7 @@ module Datadog
               o.after_set do |_, _, precedence|
                 unless precedence == Datadog::Core::Configuration::Option::Precedence::DEFAULT
                   Core.log_deprecation(key: :experimental_crash_tracking_enabled) do
-                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal '\
+                    'The profiling.advanced.experimental_crash_tracking_enabled setting has been deprecated for removal ' \
                     'and no longer does anything. Please remove it from your Datadog.configure block.'
                   end
                 end
@@ -542,6 +544,36 @@ module Datadog
               o.env 'DD_PROFILING_HEAP_CLEAN_AFTER_GC_ENABLED'
               o.default true
             end
+
+            # Controls if the profiler should use native filenames for frames in stack traces for functions implemented using
+            # native code. Setting to `false` will make the profiler fall back to default Ruby stack trace behavior (only show .rb files).
+            #
+            # @default true
+            option :native_filenames_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_NATIVE_FILENAMES_ENABLED'
+              o.default true
+            end
+
+            # Controls if the profiler should sample directly from the signal handler.
+            # Sampling directly from the signal handler improves accuracy of the data collected.
+            #
+            # We recommend using this setting with Ruby 3.2.5+ / Ruby 3.3.4+ and above
+            # as they include additional safety measures added in https://github.com/ruby/ruby/pull/11036.
+            # We have not validated it thoroughly with earlier versions, but in practice it should work on Ruby 3.0+
+            # (the key change was https://github.com/ruby/ruby/pull/3296).
+            #
+            # Enabling this on Ruby 2 is not recommended as it may cause VM crashes and/or incorrect data.
+            #
+            # @default true on Ruby 3.2.5+ / Ruby 3.3.4+, false on older Rubies
+            option :sighandler_sampling_enabled do |o|
+              o.type :bool
+              o.env 'DD_PROFILING_SIGHANDLER_SAMPLING_ENABLED'
+              o.default do
+                Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2.5') &&
+                  !(RUBY_VERSION.start_with?('3.3.') && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('3.3.4'))
+              end
+            end
           end
 
           # @public_api
@@ -570,6 +602,12 @@ module Datadog
             o.type :bool
           end
 
+          option :experimental_runtime_id_enabled do |o|
+            o.type :bool
+            o.env ['DD_TRACE_EXPERIMENTAL_RUNTIME_ID_ENABLED', 'DD_RUNTIME_METRICS_RUNTIME_ID_ENABLED']
+            o.default false
+          end
+
           option :opts, default: {}, type: :hash
           option :statsd
         end
@@ -619,38 +657,31 @@ module Datadog
           o.type :hash, nilable: true
           o.env [Core::Environment::Ext::ENV_TAGS, Core::Environment::Ext::ENV_OTEL_RESOURCE_ATTRIBUTES]
           o.env_parser do |env_value|
-            values = if env_value.include?(',')
-                       env_value.split(',')
-                     else
-                       env_value.split(' ') # rubocop:disable Style/RedundantArgument
-                     end
-            values.map! do |v|
-              v.gsub!(/\A[\s,]*|[\s,]*\Z/, '')
-
-              v.empty? ? nil : v
-            end
-
-            values.compact!
-            values.each_with_object({}) do |tag, tags|
-              key, value = tag.split(':', 2)
-              if value.nil?
-                # support tags/attributes delimited by the OpenTelemetry separator (`=`)
-                key, value = tag.split('=', 2)
-              end
-              next if value.nil? || value.empty?
-
-              # maps OpenTelemetry semantic attributes to Datadog tags
-              case key.downcase
-              when 'deployment.environment'
-                tags['env'] = value
-              when 'service.version'
-                tags['version'] = value
-              when 'service.name'
-                tags['service'] = value
-              else
-                tags[key] = value
+            # Parses a string containing key-value pairs and returns a hash.
+            # Key-value pairs are delimited by ':' OR `=`, and pairs are separated by whitespace, comma, OR BOTH.
+            result = {}
+            unless env_value.nil? || env_value.empty?
+              # falling back to comma as separator
+              sep = env_value.include?(',') ? ',' : ' '
+              # split by separator
+              env_value.split(sep).each do |tag|
+                tag.strip!
+                next if tag.empty?
+
+                # tag by : or = (for OpenTelemetry)
+                key, val = tag.split(/[:=]/, 2).map(&:strip)
+                val ||= ''
+                # maps OpenTelemetry semantic attributes to Datadog tags
+                key = case key.downcase
+                when 'deployment.environment' then 'env'
+                when 'service.version' then 'version'
+                when 'service.name' then 'service'
+                else key
+                end
+                result[key] = val unless key.empty?
               end
             end
+            result
           end
           o.setter do |new_value, old_value|
             raw_tags = new_value || {}
@@ -889,6 +920,16 @@ module Datadog
             o.env Core::Telemetry::Ext::ENV_LOG_COLLECTION
             o.default true
           end
+
+          # For internal use only.
+          # Enables telemetry debugging through the Datadog platform.
+          #
+          # @default `false`.
+          # @return [Boolean]
+          option :debug do |o|
+            o.type :bool
+            o.default false
+          end
         end
 
         # Remote configuration
@@ -958,11 +999,35 @@ module Datadog
           end
         end
 
+        # Tracer specific configuration starting with APM (e.g. DD_APM_TRACING_ENABLED).
+        # @public_api
+        settings :apm do
+          # Tracing as a transport
+          # @public_api
+          settings :tracing do
+            # Enables tracing as transport.
+            # Disabling it will set sampling priority to -1 (FORCE_DROP) on most traces,
+            # (which tells to the agent to drop these traces)
+            # except heartbeat ones (1 per minute) and manually kept ones (sampling priority to 2) (e.g. appsec events)
+            #
+            # This is different than `DD_TRACE_ENABLED`, which completely disables tracing (sends no trace at all),
+            # while this will send heartbeat traces (1 per minute) so that the service is considered alive in the backend.
+            #
+            # @default `DD_APM_TRACING_ENABLED` environment variable, otherwise `true`
+            # @return [Boolean]
+            option :enabled do |o|
+              o.env Configuration::Ext::APM::ENV_TRACING_ENABLED
+              o.default true
+              o.type :bool
+            end
+          end
+        end
+
         # TODO: Tracing should manage its own settings.
         #       Keep this extension here for now to keep things working.
         extend Datadog::Tracing::Configuration::Settings
       end
-      # rubocop:enable Metrics/BlockLength
+      # standard:enable Metrics/BlockLength
     end
   end
 end

data/lib/datadog/core/configuration/stable_config.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datadog
+  module Core
+    module Configuration
+      # Import config from config files (fleet automation)
+      module StableConfig
+        def self.extract_configuration
+          if (libdatadog_api_failure = Datadog::Core::LIBDATADOG_API_FAILURE)
+            Datadog.config_init_logger.debug("Cannot enable stable config: #{libdatadog_api_failure}")
+            return {}
+          end
+          Configurator.new.get
+        end
+
+        def self.configuration
+          @configuration ||= StableConfig.extract_configuration
+        end
+      end
+    end
+  end
+end

data/lib/datadog/core/configuration.rb

@@ -172,6 +172,10 @@ module Datadog
         end
       end
 
+      def config_init_logger
+        configuration? ? logger : logger_without_configuration
+      end
+
       # Gracefully shuts down all components.
       #
       # Components will still respond to method calls as usual,
@@ -231,19 +235,21 @@ module Datadog
         end
 
         COMPONENTS_WRITE_LOCK.synchronize do
-          begin
-            yield write_components
-          rescue ThreadError => e
-            logger_without_components.error(
-              'Detected deadlock during datadog initialization. ' \
-              'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
-              "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
-            )
-            nil
-          end
+          yield write_components
+        rescue ThreadError => e
+          logger_without_components.error(
+            'Detected deadlock during datadog initialization. ' \
+            'Please report this at https://github.com/datadog/dd-trace-rb/blob/master/CONTRIBUTING.md#found-a-bug' \
+            "\n\tSource:\n\t#{Array(e.backtrace).join("\n\t")}"
+          )
+          nil
         end
       end
 
+      def configuration?
+        (defined?(@configuration) && @configuration) != nil
+      end
+
       def components?
         # This does not need to grab the COMPONENTS_READ_LOCK because it's not returning the components
         (defined?(@components) && @components) != nil
@@ -259,12 +265,14 @@ module Datadog
         components = Components.new(settings)
 
         # Carry over state from existing components to the new ones.
-        # Currently, if we already started the remote component (which
-        # happens after a request goes through installed Rack middleware),
-        # we will start the new remote component as well.
-        old_state = {
-          remote_started: old.remote&.started?,
-        }
+        # Currently:
+        # 1. If we already started the remote component (which
+        #    happens after a request goes through installed Rack middleware),
+        #    we will start the new remote component as well.
+        # 2. If telemetry has been enabled and is enabled in the new
+        #    component tree, we send AppConfigurationChange event instead
+        #    of AppStarted event.
+        old_state = old.state
 
         old.shutdown!(components)
         components.startup!(settings, old_state: old_state)
@@ -274,6 +282,7 @@ module Datadog
       def logger_without_components
         # Use default logger without initializing components.
         # This enables logging during initialization, otherwise we'd run into deadlocks.
+
         @temp_logger ||= begin
           logger = configuration.logger.instance || Core::Logger.new($stdout)
           logger.level = configuration.diagnostics.debug ? ::Logger::DEBUG : configuration.logger.level
@@ -281,6 +290,21 @@ module Datadog
         end
       end
 
+      def logger_without_configuration
+        # There's rare cases where we need to use logger during configuration initialization,
+        # such as reading stable config. In this case we cannot access configuration.
+
+        @temp_config_logger ||= begin
+          debug_env_value = ENV[Ext::Diagnostics::ENV_DEBUG_ENABLED]&.strip&.downcase
+          debug_value = debug_env_value == 'true' || debug_env_value == '1'
+
+          logger = Core::Logger.new($stdout)
+          # We cannot access config and the default level is INFO, so we need to set the level manually
+          logger.level = debug_value ? ::Logger::DEBUG : ::Logger::INFO
+          logger
+        end
+      end
+
       # Called from our at_exit hook whenever there was a pending Interrupt exception (e.g. typically due to ctrl+c)
       # to print a nice message whenever we're taking a bit longer than usual to finish the process.
       def handle_interrupt_shutdown!