newrelic_rpm 6.13.1 → 7.2.0

data/lib/new_relic/agent/instrumentation/typhoeus/prepend.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+
+module NewRelic::Agent::Instrumentation
+  module Typhoeus
+    module Prepend
+      include NewRelic::Agent::Instrumentation::Typhoeus
+      def run(*args)
+        with_tracing { super }
+      end
+    end
+  end
+end

data/lib/new_relic/agent/javascript_instrumentor.rb

@@ -83,39 +83,44 @@ module NewRelic
         value.nil? || value.empty?
       end
 
-      def browser_timing_header #THREAD_LOCAL_ACCESS
+      def browser_timing_header(nonce=nil) #THREAD_LOCAL_ACCESS
         return '' unless js_enabled_and_ready? # fast exit
 
         state = NewRelic::Agent::Tracer.state
 
         return '' unless insert_js?(state) # slower exit
 
-        bt_config = browser_timing_config(state)
+        bt_config = browser_timing_config(state, nonce)
         return '' if bt_config.empty?
 
-        bt_config + browser_timing_loader
+        bt_config + browser_timing_loader(nonce)
       rescue => e
         ::NewRelic::Agent.logger.debug "Failure during RUM browser_timing_header construction", e
         ''
       end
 
-      def browser_timing_loader
-        html_safe_if_needed("\n<script>#{Agent.config[:js_agent_loader]}</script>")
+      def browser_timing_loader(nonce=nil)
+        html_safe_if_needed("\n<script type=\"text/javascript\"#{create_nonce(nonce)}>#{Agent.config[:js_agent_loader]}</script>")
       end
 
-      def browser_timing_config(state)
+      def browser_timing_config(state, nonce=nil)
         txn = state.current_transaction
         return '' if txn.nil?
 
         txn.freeze_name_and_execute_if_not_ignored do
           data = data_for_js_agent(txn)
           json = ::JSON.dump(data)
-          return html_safe_if_needed("\n<script>window.NREUM||(NREUM={});NREUM.info=#{json}</script>")
+          return html_safe_if_needed("\n<script type=\"text/javascript\"#{create_nonce(nonce)}>window.NREUM||(NREUM={});NREUM.info=#{json}</script>")
         end
 
         ''
       end
 
+      def create_nonce(nonce=nil)
+        return '' unless nonce
+        " nonce=\"#{nonce.to_s}\""
+      end
+
       BEACON_KEY           = "beacon".freeze
       ERROR_BEACON_KEY     = "errorBeacon".freeze
       LICENSE_KEY_KEY      = "licenseKey".freeze

data/lib/new_relic/agent/method_tracer.rb

@@ -173,30 +173,20 @@ module NewRelic
           # instrumentation into effectively one method call overhead
           # when the agent is disabled
           def assemble_code_header(method_name, metric_name_code, options)
-            header = "return #{_untraced_method_name(method_name, metric_name_code)}(*args, &block) unless NewRelic::Agent.tl_is_execution_traced?\n"
+            header = "return #{_untraced_method_name(method_name, metric_name_code)}(#{ARGS_FOR_RUBY_VERSION}) unless NewRelic::Agent.tl_is_execution_traced?\n"
             header += options[:code_header].to_s
             header
           end
 
-          # Positional and Keyword arguments are separated beginning with Ruby 2.7
-          def arguments_for_ruby_version
-            if RUBY_VERSION < "2.7.0"
-              "(*args, &block)"
-            else
-              "(*args, **kwargs, &block)"
-            end
-          end
-
           # returns an eval-able string that contains the traced
           # method code used if the agent is not creating a scope for
           # use in scoped metrics.
           def method_without_push_scope(method_name, metric_name_code, options)
-            arguments = arguments_for_ruby_version
-            "def #{_traced_method_name(method_name, metric_name_code)}#{arguments}
+            "def #{_traced_method_name(method_name, metric_name_code)}(#{ARGS_FOR_RUBY_VERSION})
               #{assemble_code_header(method_name, metric_name_code, options)}
               t0 = Time.now
               begin
-                #{_untraced_method_name(method_name, metric_name_code)}#{arguments}\n
+                #{_untraced_method_name(method_name, metric_name_code)}(#{ARGS_FOR_RUBY_VERSION})\n
               ensure
                 duration = (Time.now - t0).to_f
                 NewRelic::Agent.record_metric(\"#{metric_name_code}\", duration)
@@ -208,12 +198,11 @@ module NewRelic
           # returns an eval-able string that contains the tracing code
           # for a fully traced metric including scoping
           def method_with_push_scope(method_name, metric_name_code, options)
-            arguments = arguments_for_ruby_version
-            "def #{_traced_method_name(method_name, metric_name_code)}#{arguments}
+            "def #{_traced_method_name(method_name, metric_name_code)}(#{ARGS_FOR_RUBY_VERSION})
               #{options[:code_header]}
               result = ::NewRelic::Agent::MethodTracerHelpers.trace_execution_scoped(\"#{metric_name_code}\",
                         :metric => #{options[:metric]}) do
-                #{_untraced_method_name(method_name, metric_name_code)}#{arguments}
+                #{_untraced_method_name(method_name, metric_name_code)}(#{ARGS_FOR_RUBY_VERSION})
               end
               #{options[:code_footer]}
               result
@@ -337,6 +326,7 @@ module NewRelic
           class_eval traced_method, __FILE__, __LINE__
           alias_method _untraced_method_name(method_name, metric_name_code), method_name
           alias_method method_name, _traced_method_name(method_name, metric_name_code)
+          ruby2_keywords(_traced_method_name(method_name, metric_name_code)) if respond_to?(:ruby2_keywords, true)
           send visibility, method_name
           send visibility, _traced_method_name(method_name, metric_name_code)
           ::NewRelic::Agent.logger.debug("Traced method: class = #{derived_class_name},"+

data/lib/new_relic/agent/new_relic_service.rb

@@ -41,7 +41,6 @@ module NewRelic
         @configured_collector = collector
         @request_timeout = Agent.config[:timeout]
         @ssl_cert_store = nil
-        @use_bundled_certs = false
         @in_session = nil
         @agent_id = nil
         @shared_tcp_connection = nil
@@ -299,7 +298,7 @@ module NewRelic
       end
 
       def set_cert_store(conn)
-        if @use_bundled_certs || NewRelic::Agent.config[:ca_bundle_path]
+        if NewRelic::Agent.config[:ca_bundle_path]
           conn.cert_store  = ssl_cert_store
         else
           ::NewRelic::Agent.logger.debug("Using default security certificates")
@@ -348,13 +347,8 @@ module NewRelic
         start_connection(conn)
         conn
       rescue Timeout::Error
-        if @use_bundled_certs == false
-          ::NewRelic::Agent.logger.info("Unable to connect. Falling back to bundled security certificates")
-          @use_bundled_certs = true
-          retry
-        else
-          raise
-        end
+        ::NewRelic::Agent.logger.info ("Timeout while attempting to connect. You may need to install system-level CA Certificates, as the ruby agent no longer includes these.")
+        raise
       end
 
       # The path to the certificate file used to verify the SSL
@@ -363,9 +357,6 @@ module NewRelic
         if path_override = NewRelic::Agent.config[:ca_bundle_path]
           NewRelic::Agent.logger.warn("Couldn't find CA bundle from configured ca_bundle_path: #{path_override}") unless File.exist? path_override
           path_override
-        else
-          ::NewRelic::Agent.increment_metric("Supportability/Ruby/Certificate/BundleRequired")
-          File.expand_path(File.join(control.newrelic_root, 'cert', 'cacert.pem'))
         end
       end
 
@@ -435,7 +426,18 @@ module NewRelic
         size = data.size
 
         # Preconnect needs to always use the configured collector host, not the redirect host
-        endpoint_specific_collector = (method == :preconnect) ? @configured_collector : @collector
+        # endpoint_specific_collector = (method == :preconnect) ? @configured_collector : @collector
+
+        # This is a temporary workaround due to errors occurring on the staging collector. The prod collector does not have the same issue.
+        # The staging collector does not respond correctly when using the configured collector host for preconnect, so must use the redirect host
+        # Once this issue is resolved on the staging collector, use the original line that is commented out above.
+        endpoint_specific_collector = if method == :preconnect && (@configured_collector && @configured_collector.name != 'staging-collector.newrelic.com')
+                                        ::NewRelic::Agent.logger.debug "Using configured collector for preconnect: #{@configured_collector}"
+                                        @configured_collector
+                                      else
+                                        ::NewRelic::Agent.logger.debug "Using redirect host for collector: #{@collector}"
+                                        @collector
+                                      end
 
         uri = remote_method_uri(method)
         full_uri = "#{endpoint_specific_collector}#{uri}"
@@ -526,6 +528,7 @@ module NewRelic
         else
           request = Net::HTTP::Post.new(opts[:uri], headers)
         end
+        @audit_logger.log_request_headers(opts[:uri], headers)
         request['user-agent'] = user_agent
         request.content_type = "application/octet-stream"
         request.body = opts[:data]

data/lib/new_relic/agent/samplers/memory_sampler.rb

@@ -29,7 +29,7 @@ module NewRelic
             end
           elsif platform =~ /darwin9/ # 10.5
             @sampler = ShellPS.new("ps -o rsz")
-          elsif platform =~ /darwin1\d+/ # >= 10.6
+          elsif platform =~ /darwin(1|2)\d+/ # >= 10.6
             @sampler = ShellPS.new("ps -o rss")
           elsif platform =~ /freebsd/
             @sampler = ShellPS.new("ps -o rss")

data/lib/new_relic/agent/span_event_primitive.rb

@@ -165,16 +165,18 @@ module NewRelic
         end
       end
 
+      def merge_and_freeze_attributes agent_attributes, error_attributes
+        return agent_attributes.freeze unless error_attributes
+        return error_attributes.freeze if agent_attributes.equal?(NewRelic::EMPTY_HASH)
+        agent_attributes.merge!(error_attributes).freeze
+      end
+
       def agent_attributes segment
-        attributes = segment.attributes
-        agent_attributes = attributes.agent_attributes_for(NewRelic::Agent::AttributeFilter::DST_SPAN_EVENTS)
+        agent_attributes = segment.attributes
+          .agent_attributes_for(NewRelic::Agent::AttributeFilter::DST_SPAN_EVENTS)
         error_attributes = error_attributes(segment)
-        if agent_attributes || error_attributes
-          agent_attributes.merge!(error_attributes) if error_attributes
-          agent_attributes.freeze
-        else
-          NewRelic::EMPTY_HASH
-        end
+        return NewRelic::EMPTY_HASH unless agent_attributes || error_attributes
+        merge_and_freeze_attributes(agent_attributes, error_attributes)
       end
 
       def parent_guid segment

data/lib/new_relic/agent/sql_sampler.rb

@@ -65,7 +65,7 @@ module NewRelic
       end
 
       # This is called when we are done with the transaction.
-      def on_finishing_transaction(state, name, time=Time.now)
+      def on_finishing_transaction(state, name)
         return unless enabled?
 
         data = state.sql_sampler_transaction_data
@@ -359,10 +359,10 @@ module NewRelic
       # need to hash the same way in every process, to be able to aggregate slow SQL traces
       def consistent_hash(string)
         if NewRelic::Agent.config[:'slow_sql.use_longer_sql_id']
-          Digest::MD5.hexdigest(string).hex.modulo(2**63-1)
+          Digest::SHA1.hexdigest(string).hex.modulo(2**63-1)
         else
           # from when sql_id needed to fit in an INT(11)
-          Digest::MD5.hexdigest(string).hex.modulo(2**31-1)
+          Digest::SHA1.hexdigest(string).hex.modulo(2**31-1)
         end
       end
     end

data/lib/new_relic/agent/stats_engine/gc_profiler.rb

@@ -99,7 +99,7 @@ module NewRelic
 
           # When using GC::Profiler, it's important to periodically call
           # GC::Profiler.clear in order to avoid unbounded growth in the number
-          # of GC recordds that are stored. However, we actually do this
+          # of GC records that are stored. However, we actually do this
           # internally within MonotonicGCProfiler on calls to #total_time_s,
           # so the reset here is a no-op.
           def reset; end

data/lib/new_relic/agent/transaction.rb

@@ -464,7 +464,7 @@ module NewRelic
         @ignore_apdex = options[:ignore_apdex] if options.key? :ignore_apdex
         @ignore_enduser = options[:ignore_enduser] if options.key? :ignore_enduser
 
-        nest_initial_segment if nesting_max_depth == 1
+        nest_initial_segment if segments.length == 1
         nested_name = self.class.nested_transaction_name options[:transaction_name]
         segment = create_segment nested_name
         set_default_transaction_name(options[:transaction_name], category)
@@ -571,9 +571,6 @@ module NewRelic
                                AttributeFilter::DST_ERROR_COLLECTOR
 
         if http_response_code
-          add_agent_attribute(:httpResponseCode, http_response_code.to_s, default_destinations)
-          # Sending status code as an int with http.statusCode key is correct
-          # The above attribute is deprecated and should be removed in agent version 7.0.0
           add_agent_attribute(:'http.statusCode', http_response_code, default_destinations)
         end
 

data/lib/new_relic/agent/transaction/abstract_segment.rb

@@ -167,7 +167,7 @@ module NewRelic
 
           if finished?
             transaction.async = true
-            parent.descendant_complete self, segment
+            parent.descendant_complete(self, segment) if parent
           end
         end
 

data/lib/new_relic/agent/transaction/distributed_tracer.rb

@@ -68,13 +68,19 @@ module NewRelic
             payload
         end
 
-        def insert_headers request
-          insert_trace_context_header request
-          insert_distributed_trace_header request
-          insert_cross_app_header request
+        def log_request_headers headers, direction = "OUTGOING"
+          NewRelic::Agent.logger.debug "#{direction} REQUEST HEADERS: #{headers}"
+        end
+
+        def insert_headers headers
+          insert_trace_context_header headers
+          insert_distributed_trace_header headers
+          insert_cross_app_header headers
+          log_request_headers headers
         end
 
         def consume_message_headers headers, tracer_state, transport_type
+          log_request_headers headers, "INCOMING"
           consume_message_distributed_tracing_headers headers, transport_type
           consume_message_cross_app_tracing_headers headers, tracer_state
           consume_message_synthetics_headers headers
@@ -90,11 +96,11 @@ module NewRelic
           end
         end
 
-        def insert_distributed_trace_header request
+        def insert_distributed_trace_header headers
           return unless Agent.config[:'distributed_tracing.enabled']
           return if Agent.config[:'exclude_newrelic_header']
           payload = create_distributed_trace_payload
-          request[NewRelic::NEWRELIC_KEY] = payload.http_safe if payload
+          headers[NewRelic::NEWRELIC_KEY] = payload.http_safe if payload
         end
 
         def insert_cat_headers headers

data/lib/new_relic/agent/transaction/message_broker_segment.rb

@@ -93,6 +93,7 @@ module NewRelic
           if headers && transaction && action == :produce && record_metrics?
             transaction.distributed_tracer.insert_distributed_trace_header headers
             transaction.distributed_tracer.insert_cat_headers headers
+            transaction.distributed_tracer.log_request_headers headers
           end
         rescue => e
           NewRelic::Agent.logger.error "Error during message header processing", e

data/lib/new_relic/agent/vm/mri_vm.rb

@@ -45,9 +45,11 @@ module NewRelic
 
         def gather_ruby_vm_stats(snap)
           if supports?(:method_cache_invalidations)
-            vm_stats = RubyVM.stat
-            snap.method_cache_invalidations = vm_stats[:global_method_state]
-            snap.constant_cache_invalidations = vm_stats[:global_constant_state]
+            snap.method_cache_invalidations = RubyVM.stat[:global_method_state]
+          end
+
+          if supports?(:constant_cache_invalidations)
+            snap.constant_cache_invalidations = RubyVM.stat[:global_constant_state]
           end
         end
 
@@ -66,7 +68,7 @@ module NewRelic
           when :minor_gc_count
             RUBY_VERSION >= '2.1.0'
           when :method_cache_invalidations
-            RUBY_VERSION >= '2.1.0'
+            RUBY_VERSION >= '2.1.0' && RUBY_VERSION < '3.0.0'
           when :constant_cache_invalidations
             RUBY_VERSION >= '2.1.0'
           else

data/lib/new_relic/cli/commands/deployments.rb

@@ -7,7 +7,6 @@
 
 require 'yaml'
 require 'net/http'
-require 'rexml/document'
 require 'new_relic/agent/hostname'
 
 # We need to use the Control object but we don't want to load

data/lib/new_relic/constants.rb

@@ -28,6 +28,10 @@ module NewRelic
   TRACEPARENT_KEY = "traceparent"
   TRACESTATE_KEY = "tracestate"
 
+  # Right now, old and new Rubies are the same.  Intention is to use "(...)" for
+  # Ruby 2.7+ for argument delegation, but doing so is a breaking API change.
+  ARGS_FOR_RUBY_VERSION = RUBY_VERSION < "2.7.0" ? "*args, &block" : "*args, &block"
+
   HTTP_TRACEPARENT_KEY = "HTTP_#{TRACEPARENT_KEY.upcase}"
   HTTP_TRACESTATE_KEY = "HTTP_#{TRACESTATE_KEY.upcase}"
   HTTP_NEWRELIC_KEY = "HTTP_#{NEWRELIC_KEY.upcase}"

data/lib/new_relic/control/frameworks/rails.rb

@@ -82,15 +82,17 @@ module NewRelic
         end
 
         def install_browser_monitoring(config)
-          return if defined?(@browser_monitoring_installed) && @browser_monitoring_installed
-          @browser_monitoring_installed = true
-          return if config.nil? || !config.respond_to?(:middleware) || !Agent.config[:'browser_monitoring.auto_instrument']
-          begin
-            require 'new_relic/rack/browser_monitoring'
-            config.middleware.use NewRelic::Rack::BrowserMonitoring
-            ::NewRelic::Agent.logger.debug("Installed New Relic Browser Monitoring middleware")
-          rescue => e
-            ::NewRelic::Agent.logger.warn("Error installing New Relic Browser Monitoring middleware", e)
+          @install_lock.synchronize do
+            return if defined?(@browser_monitoring_installed) && @browser_monitoring_installed
+            @browser_monitoring_installed = true
+            return if config.nil? || !config.respond_to?(:middleware) || !Agent.config[:'browser_monitoring.auto_instrument']
+            begin
+              require 'new_relic/rack/browser_monitoring'
+              config.middleware.use NewRelic::Rack::BrowserMonitoring
+              ::NewRelic::Agent.logger.debug("Installed New Relic Browser Monitoring middleware")
+            rescue => e
+              ::NewRelic::Agent.logger.warn("Error installing New Relic Browser Monitoring middleware", e)
+            end
           end
         end
 

data/lib/new_relic/control/instance_methods.rb

@@ -165,6 +165,7 @@ module NewRelic
       protected
 
       def initialize(local_env, config_file_override=nil)
+        @install_lock = Mutex.new
         @local_env = local_env
         @started_in_env = nil
 

data/lib/new_relic/dependency_detection.rb

@@ -21,6 +21,7 @@ module DependencyDetection
     end
 
     @items << item
+    return item
   end
 
   def detect!
@@ -51,28 +52,74 @@ module DependencyDetection
   class Dependent
     attr_reader :executed
     attr_accessor :name
+    attr_writer :config_name
+    attr_reader :dependencies
+    attr_reader :prepend_conflicts
+
+
     def executed!
       @executed = true
     end
 
-    attr_reader :dependencies
+    def config_name
+      @config_name || @name
+    end
 
     def initialize
       @dependencies = []
       @executes = []
+      @prepend_conflicts = []
       @name = nil
+      @config_name = nil
     end
 
     def dependencies_satisfied?
       !executed and check_dependencies
     end
 
+    def source_location_for klass, method_name
+      Object.instance_method(:method).bind(klass.allocate).call(method_name).source_location.to_s
+    end
+
+    # Extracts the instrumented library name from the instrumenting module's name
+    # Given "NewRelic::Agent::Instrumentation::NetHTTP::Prepend"
+    # Will extract "NetHTTP" which is in the 2nd to last spot
+    def extract_supportability_name instrumenting_module
+      instrumenting_module.to_s.split("::")[-2]
+    end
+
+    def log_and_instrument method, instrumenting_module, supportability_name
+      supportability_name ||= extract_supportability_name(instrumenting_module)
+      NewRelic::Agent.logger.info "Installing New Relic supported #{supportability_name} instrumentation using #{method}"
+      NewRelic::Agent.record_metric("Supportability/Instrumentation/#{supportability_name}/#{method}", 0.0)
+      yield
+    end
+
+    def prepend_instrument target_class, instrumenting_module, supportability_name=nil
+      log_and_instrument("Prepend", instrumenting_module, supportability_name) do
+        target_class.send :prepend, instrumenting_module
+      end
+    end
+
+    def chain_instrument instrumenting_module, supportability_name=nil
+      log_and_instrument("MethodChaining", instrumenting_module, supportability_name) do
+        instrumenting_module.instrument!
+      end
+    end
+
+    def chain_instrument_target target, instrumenting_module, supportability_name=nil
+      NewRelic::Agent.logger.info "Installing deferred #{target} instrumentation"
+      log_and_instrument("MethodChaining", instrumenting_module, supportability_name) do
+        instrumenting_module.instrument! target
+      end
+    end
+
     def execute
       @executes.each do |x|
         begin
           x.call
         rescue => err
-          NewRelic::Agent.logger.error( "Error while installing #{self.name} instrumentation:", err )
+          NewRelic::Agent.logger.error "Error while installing #{self.name} instrumentation:", err
           break
         end
       end
@@ -98,24 +145,87 @@ module DependencyDetection
     end
 
     def allowed_by_config?
-      # If we don't have a name, can't check config so allow it
-      return true if self.name.nil?
+      !(disabled_configured? || deprecated_disabled_configured?)
+    end
+
+    # TODO: Remove in 8.0
+    # will only return true if a disabled key is found and is truthy
+    def deprecated_disabled_configured?
+      return false if self.name.nil?
 
       key = "disable_#{self.name}".to_sym
-      if (::NewRelic::Agent.config[key] == true)
-        ::NewRelic::Agent.logger.debug("Not installing #{self.name} instrumentation because of configuration #{key}")
-        false
-      else
-        true
+      return false unless ::NewRelic::Agent.config[key] == true
+
+      ::NewRelic::Agent.logger.debug("Not installing #{self.name} instrumentation because of configuration #{key}")
+      ::NewRelic::Agent.logger.debug \
+        "[DEPRECATED] configuration #{key} for #{self.name} will be removed in the next major release." \
+        " Use `#{config_key}` with one of `#{VALID_CONFIG_VALUES.map(&:to_s).inspect}`"
+
+      return true
+    end
+
+    def config_key
+      return nil if self.config_name.nil?
+      @config_key ||= "instrumentation.#{self.config_name}".to_sym
+    end
+
+    VALID_CONFIG_VALUES = [:auto, :disabled, :prepend, :chain]
+    AUTO_CONFIG_VALUE = VALID_CONFIG_VALUES[0]
+
+    VALID_CONFIG_VALUES.each do |value|
+      define_method "#{value}_configured?" do
+        value == config_value
       end
     end
 
+    # returns only a valid value for instrumentation configuration
+    # If user uses "enabled" it's converted to "auto"
+    def valid_config_value retrieved_value
+      VALID_CONFIG_VALUES.include?(retrieved_value) ? retrieved_value : AUTO_CONFIG_VALUE
+    end
+
+    # fetches and transform potentially invalid value given to one of the valid config values
+    # logs the resolved value during debug mode.
+    def fetch_config_value(key)
+      valid_value = valid_config_value(::NewRelic::Agent.config[key].to_s.to_sym)
+      ::NewRelic::Agent.logger.debug("Using #{valid_value} configuration value for #{self.name} to configure instrumentation")
+      return valid_value
+    end
+
+    def config_value
+      return AUTO_CONFIG_VALUE unless config_key
+      fetch_config_value(config_key)
+    end
+
     def named(new_name)
       self.name = new_name
     end
 
+    def configure_with(new_config_name)
+      self.config_name = new_config_name
+    end
+
     def executes &block
       @executes << block if block_given?
     end
+
+    def conflicts_with_prepend &block
+      @prepend_conflicts << block if block_given?
+    end
+
+    def use_prepend?
+      prepend_configured? || (auto_configured? && !prepend_conflicts?)
+    end
+
+    def prepend_conflicts?
+      @prepend_conflicts.any? do |conflict|
+        begin
+          conflict.call
+        rescue => err
+          NewRelic::Agent.logger.error( "Error while checking prepend conflicts #{self.name}:", err )
+          false # assumes no conflicts exist since `prepend` is preferred method of instrumenting
+        end
+      end
+    end
   end
 end