newrelic_rpm 5.3.0.346 → 5.4.0.347

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a104aa20426e4da3129e97dfaa651e7232ef804cf91de87b0aa4ac68d00d3fe7
-  data.tar.gz: f8ae7ba61e0dc5442212c8d4a8d985d4e9ac353c23e9ff61ebb709ff11bfd28a
+  metadata.gz: 979f362569155df00b770dba3907bd0eb894169f70b712085ae19d16852f53a7
+  data.tar.gz: b67e7d1c0cb1b5831f06d29c574f8efea5a95405a809b97c55add2e40d0cb101
 SHA512:
-  metadata.gz: 58d78970fa151543b165d1ef8b37a9ab7adefabf35ccad2292df5038812a1d5d5524f21cb7822a8d99bc4eabed8ceafc537b6f913b829f2a25ff0cf19a399e3a
-  data.tar.gz: 6859b4ba6ceaab1faeac5634ea264e015329f20af436872da764e3f17ac6b22f2ad2cbfc4604eccc883d9f7b1a0ef5c1fb193c9faf7fdbfab107bae3a2109b8d
+  metadata.gz: 2974f98ab0c47eb9f1c0e2014fb4122138fe4d1bf4d5265f8d09d1583f7ee8fc3f9ab5cb86bf1dc7e31d8b44d5762e56c8e1edd421120151ae85e3e2e4bd854d
+  data.tar.gz: 7d16296c46075e1e84824afaeff67db60ea39e0b5b4032bbae9bbcbde8c6c5ca6dc6d40618c92cbf29391f0ba36876937e8d2444313d3805031827a7e8313607

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # New Relic Ruby Agent Release Notes #
 
+## v5.4.0
+
+  * Capacity analysis for multi-threaded dispatchers
+
+    Metrics around capacity analysis did not previously account for multi-threaded
+    dispatchers, and consequently could result in capacities of over 100% being
+    recorded. This version now properly accounts for multi-threaded dispatchers.
+
+  * `NewRelic::Agent.disable_transaction_tracing` deprecated
+
+    `NewRelic::Agent.disable_transaction_tracing` has been deprecated. Users
+    are encouraged to use `NewRelic::Agent.disable_all_tracing` or
+    `NewRelic::Agent.ignore_transaction` instead.
+
+  * Bugfix for SQL over-obfuscation
+
+    A bug, introduced in v5.3.0, where SQL could be over-obfuscated for some
+    database adapters has been fixed.
+
+  * Bugfix for span event data in Resque processes
+
+    A bug where span events would not be sent from Resque processes due to a
+    missing endpoint has been fixed.
+
 ## v5.3.0 ##
 
   * Distributed Tracing

data/config.dot

@@ -42,7 +42,7 @@ digraph AgentEnabled {
   "[dispatcher]"
   "[force_send]"
   "[textmate]"
-  "[post_size_limit]"
+  "[max_payload_size_in_bytes]"
   "[sync_startup]"
   "[send_data_on_exit]"
   "[high_security]"
@@ -223,7 +223,7 @@ digraph AgentEnabled {
   "NewRelicService#initialize" -> "[timeout]"
   "NewRelicService#initialize" -> "[license_key]"
   "NewRelicService#initialize" -> "Control#server"
-  "NewRelicService#check_post_size" -> "[post_size_limit]"
+  "NewRelicService#check_post_size" -> "[max_payload_size_in_bytes]"
   "NewRelicService#connect" -> "Control#server_from_host"
   "NewRelicService#send_request" -> "Control#http_connection"
 

data/lib/new_relic/agent.rb

@@ -45,7 +45,6 @@ module NewRelic
     require 'new_relic/agent/sql_sampler'
     require 'new_relic/agent/commands/thread_profiler_session'
     require 'new_relic/agent/error_collector'
-    require 'new_relic/agent/busy_calculator'
     require 'new_relic/agent/sampler'
     require 'new_relic/agent/database'
     require 'new_relic/agent/datastores'
@@ -55,6 +54,7 @@ module NewRelic
     require 'new_relic/agent/http_clients/uri_util'
     require 'new_relic/agent/system_info'
     require 'new_relic/agent/external'
+    require 'new_relic/agent/deprecator'
 
     require 'new_relic/agent/instrumentation/controller_instrumentation'
 
@@ -206,11 +206,17 @@ module NewRelic
     # This method is safe to use from any thread.
     #
     # @api public
+
+    SUPPORTABILITY_INCREMENT_METRIC = 'Supportability/API/increment_metric'.freeze
+
     def increment_metric(metric_name, amount=1) #THREAD_LOCAL_ACCESS
       return unless agent
-
-      { 'Supportability/API/increment_metric' => 1,  metric_name => amount }.each do |metric, increment_amount|
-        agent.stats_engine.tl_record_unscoped_metrics(metric) {|stats| stats.increment_count(increment_amount) }
+      if amount == 1
+        metrics = [metric_name, SUPPORTABILITY_INCREMENT_METRIC]
+        agent.stats_engine.tl_record_unscoped_metrics(metrics) {|stats| stats.increment_count}
+      else
+        agent.stats_engine.tl_record_unscoped_metrics(metric_name) {|stats| stats.increment_count(amount)}
+        agent.stats_engine.tl_record_unscoped_metrics(SUPPORTABILITY_INCREMENT_METRIC) {|stats| stats.increment_count}
       end
     end
 
@@ -504,16 +510,10 @@ module NewRelic
     # @api public
     #
     def disable_transaction_tracing
+      Deprecator.deprecate :disable_transaction_tracing,
+                           'disable_all_tracing or ignore_transaction'
       record_api_supportability_metric(:disable_transaction_tracing)
-
-      return yield unless agent
-
-      state = agent.set_record_tt(false)
-      begin
-        yield
-      ensure
-        agent.set_record_tt(state)
-      end
+      yield
     end
 
     # This method sets the state of sql recording in the transaction
@@ -547,12 +547,6 @@ module NewRelic
       NewRelic::Agent::TransactionState.tl_get.is_execution_traced?
     end
 
-    # helper method to check the thread local to determine whether the
-    # transaction in progress is traced or not
-    def tl_is_transaction_traced?
-      NewRelic::Agent::TransactionState.tl_get.is_transaction_traced?
-    end
-
     # helper method to check the thread local to determine whether sql
     # is being recorded or not
     def tl_is_sql_recorded?

data/lib/new_relic/agent/agent.rb

@@ -268,16 +268,6 @@ module NewRelic
           prev.nil? || prev
         end
 
-        # Sets a thread local variable as to whether we should or
-        # should not record transaction traces in the current
-        # thread. Returns the previous value, if there is one
-        def set_record_tt(should_record) #THREAD_LOCAL_ACCESS
-          state = TransactionState.tl_get
-          prev = state.record_tt
-          state.record_tt = should_record
-          prev.nil? || prev
-        end
-
         # Push flag indicating whether we should be tracing in this
         # thread. This uses a stack which allows us to disable tracing
         # children of a transaction without affecting the tracing of
@@ -934,6 +924,7 @@ module NewRelic
           when :error_event_data        then @error_collector.error_event_aggregator
           when :analytic_event_data     then transaction_event_aggregator
           when :custom_event_data       then @custom_event_aggregator
+          when :span_event_data         then span_event_aggregator
           when :sql_trace_data          then @sql_sampler
           end
         end
@@ -1084,7 +1075,7 @@ module NewRelic
         end
 
         def harvest_and_send_timeslice_data
-          NewRelic::Agent::BusyCalculator.harvest_busy
+          NewRelic::Agent::TransactionTimeAggregator.harvest!
           harvest_and_send_from_container(@stats_engine, :metric_data)
         end
 

data/lib/new_relic/agent/configuration/default_source.rb

@@ -504,8 +504,8 @@ module NewRelic
           :allowed_from_server => false,
           :description => 'If <code>true</code>, enables the exit handler that sends data to the New Relic <a href="https://docs.newrelic.com/docs/apm/new-relic-apm/getting-started/glossary#collector">collector</a> before shutting down.'
         },
-        :post_size_limit => {
-          :default => 2 * 1024 * 1024, # 2MB
+        :max_payload_size_in_bytes => {
+          :default => 1000000,
           :public => false,
           :type => Integer,
           :allowed_from_server => true,

data/lib/new_relic/agent/configuration/server_source.rb

@@ -25,6 +25,7 @@ module NewRelic
           "error_beacon",
           "js_agent_file",
           "js_agent_loader",
+          "max_payload_size_in_bytes",
           "primary_application_id",
           "sampling_target",
           "sampling_target_period_in_seconds",

data/lib/new_relic/agent/database.rb

@@ -203,7 +203,7 @@ module NewRelic
         def safe_sql
           @safe_sql ||= case Database.record_sql_method
             when :obfuscated
-              Database.obfuscate_sql(sql)
+              Database.obfuscate_sql(self)
             when :raw
               sql.to_s
             else

data/lib/new_relic/agent/javascript_instrumentor.rb

@@ -65,9 +65,6 @@ module NewRelic
         if !state.current_transaction
           ::NewRelic::Agent.logger.debug "Not in transaction. Skipping browser instrumentation."
           false
-        elsif !state.is_transaction_traced?
-          ::NewRelic::Agent.logger.debug "Transaction is not traced. Skipping browser instrumentation."
-          false
         elsif !state.is_execution_traced?
           ::NewRelic::Agent.logger.debug "Execution is not traced. Skipping browser instrumentation."
           false

data/lib/new_relic/agent/new_relic_service.rb

@@ -193,7 +193,7 @@ module NewRelic
       # We do not compress if content is smaller than 64kb.  There are
       # problems with bugs in Ruby in some versions that expose us
       # to a risk of segfaults if we compress aggressively.
-      def compress_request_if_needed(data)
+      def compress_request_if_needed(data, endpoint)
         encoding = 'identity'
         if data.size > 64 * 1024
           encoding = Agent.config[:compressed_content_encoding]
@@ -203,7 +203,7 @@ module NewRelic
             Encoders::Compressed::Deflate.encode(data)
           end
         end
-        check_post_size(data)
+        check_post_size(data, endpoint)
         [data, encoding]
       end
 
@@ -412,7 +412,7 @@ module NewRelic
         end
         serialize_finish_ts = Time.now
 
-        data, encoding = compress_request_if_needed(data)
+        data, encoding = compress_request_if_needed(data, method)
         size = data.size
 
         uri = remote_method_uri(method)
@@ -471,9 +471,10 @@ module NewRelic
 
       # Raises an UnrecoverableServerException if the post_string is longer
       # than the limit configured in the control object
-      def check_post_size(post_string)
-        return if post_string.size < Agent.config[:post_size_limit]
+      def check_post_size(post_string, endpoint)
+        return if post_string.size < Agent.config[:max_payload_size_in_bytes]
         ::NewRelic::Agent.logger.debug "Tried to send too much data: #{post_string.size} bytes"
+        NewRelic::Agent.increment_metric("Supportability/Agent/Collector/#{endpoint}/MaxPayloadSizeLimit")
         raise UnrecoverableServerException.new('413 Request Entity Too Large')
       end
 

data/lib/new_relic/agent/pipe_service.rb

@@ -53,6 +53,10 @@ module NewRelic
         write_to_pipe(:error_event_data, events) if events
       end
 
+      def span_event_data(events)
+        write_to_pipe(:span_event_data, events) if events
+      end
+
       def sql_trace_data(sql)
         write_to_pipe(:sql_trace_data, sql) if sql
       end

data/lib/new_relic/agent/stats_engine.rb

@@ -186,7 +186,7 @@ module NewRelic
       # For use by test code only.
       def clear_stats
         reset!
-        NewRelic::Agent::BusyCalculator.reset
+        NewRelic::Agent::TransactionTimeAggregator.reset!
       end
 
       # For use by test code only.

data/lib/new_relic/agent/transaction.rb

@@ -11,6 +11,7 @@ require 'new_relic/agent/transaction/request_attributes'
 require 'new_relic/agent/transaction/tracing'
 require 'new_relic/agent/transaction/distributed_tracing'
 require 'new_relic/agent/cross_app_tracing'
+require 'new_relic/agent/transaction_time_aggregator'
 
 module NewRelic
   module Agent
@@ -446,7 +447,7 @@ module NewRelic
 
         sql_sampler.on_start_transaction(state, start_time, request_path)
         NewRelic::Agent.instance.events.notify(:start_transaction)
-        NewRelic::Agent::BusyCalculator.dispatcher_start(start_time)
+        NewRelic::Agent::TransactionTimeAggregator.transaction_start(start_time)
 
         ignore! if user_defined_rules_ignore?
 
@@ -542,7 +543,7 @@ module NewRelic
 
         outermost_frame.finish
 
-        NewRelic::Agent::BusyCalculator.dispatcher_finish(end_time)
+        NewRelic::Agent::TransactionTimeAggregator.transaction_stop(end_time)
 
         commit!(state, end_time, outermost_frame.name) unless @ignore_this_transaction
       end

data/lib/new_relic/agent/transaction_state.rb

@@ -34,13 +34,12 @@ module NewRelic
       def initialize
         @untraced = []
         @current_transaction = nil
-        @record_tt = nil
         @record_sql = nil
       end
 
       # This starts the timer for the transaction.
       def reset(transaction=nil)
-        # We purposefully don't reset @untraced, @record_tt and @record_sql
+        # We purposefully don't reset @untraced or @record_sql
         # since those are managed by NewRelic::Agent.disable_* calls explicitly
         # and (more importantly) outside the scope of a transaction
 
@@ -125,11 +124,7 @@ module NewRelic
       end
 
       # TT's and SQL
-      attr_accessor :record_tt, :record_sql
-
-      def is_transaction_traced?
-        @record_tt != false
-      end
+      attr_accessor :record_sql
 
       def is_sql_recorded?
         @record_sql != false

data/lib/new_relic/agent/transaction_time_aggregator.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
+
+require 'objspace'
+
+# This module powers the Busy calculation for the Capacity report in
+# APM (https://rpm.newrelic.com/accounts/.../applications/.../optimize/capacity_analysis).
+#
+#               total time spent in transactions this harvest across all threads
+# Busy time = ------------------------------------------------------------------------
+#             (elapsed time for this harvest cycle) * (# threads that had transactions)
+#
+module NewRelic
+  module Agent
+    module TransactionTimeAggregator
+      TransactionStats = Struct.new :transaction_started_at, :elapsed_transaction_time
+
+      @lock                     = Mutex.new
+      @harvest_cycle_started_at = Time.now
+
+      @stats = Hash.new do |h, k|
+        h[k] = TransactionStats.new nil, 0.0
+      end
+
+      def reset!(now = Time.now)
+        @harvest_cycle_started_at = now
+        @stats.clear
+      end
+
+      def transaction_start(now = Time.now)
+        @lock.synchronize do
+          set_transaction_start_time now
+        end
+      end
+
+      def transaction_stop(now = Time.now)
+        @lock.synchronize do
+          record_elapsed_transaction_time_until now
+          set_transaction_start_time nil
+        end
+      end
+
+      INSTANCE_BUSY_METRIC = 'Instance/Busy'.freeze
+
+      def harvest!(now = Time.now)
+        active_threads = 0
+        result = @lock.synchronize do
+          # Sum up the transaction times spent in each thread
+          elapsed_transaction_time = @stats.inject(0.0) do |total, (thread_id, entry)|
+            total + transaction_time_in_thread(thread_id, entry, now)
+          end
+
+          active_threads = @stats.size
+          elapsed_harvest_time      = (now - @harvest_cycle_started_at) * active_threads
+          @harvest_cycle_started_at = now
+
+          # Clear out the stats for all threads, _except_ the live ones
+          # that have transactions still open (we'll count the rest of
+          # those in a future harvest)
+          @stats.keep_if do |thread_id, _|
+            in_transaction?(thread_id) && thread_is_alive?(thread_id)
+          end
+
+          if elapsed_harvest_time > 0.0
+            elapsed_transaction_time / elapsed_harvest_time
+          else
+            0.0
+          end
+        end
+
+        if Agent.config[:report_instance_busy]
+          NewRelic::Agent.record_metric(INSTANCE_BUSY_METRIC, result)
+        end
+
+        result
+      end
+
+      module_function :reset!,
+                      :transaction_start,
+                      :transaction_stop,
+                      :harvest!
+
+      class <<self
+        private
+
+        def record_elapsed_transaction_time_until(timestamp, thread_id: current_thread)
+          @stats[thread_id].elapsed_transaction_time +=
+            (timestamp - (@stats[thread_id].transaction_started_at || 0.0))
+        end
+
+        def in_transaction?(thread_id = current_thread)
+          !!@stats[thread_id].transaction_started_at
+        end
+
+        def current_thread
+          Thread.current.object_id
+        end
+
+        def thread_is_alive?(thread_id)
+          thread = ObjectSpace._id2ref(thread_id)
+          thread && thread.alive?
+        rescue StandardError
+          false
+        end
+
+        def set_transaction_start_time(timestamp)
+          @stats[current_thread].transaction_started_at = timestamp
+        end
+
+        def split_transaction_at_harvest(now, thread_id: nil)
+          raise ArgumentError, 'thread_id required' unless thread_id
+          @stats[thread_id].transaction_started_at = now
+          @stats[thread_id].elapsed_transaction_time = 0.0
+        end
+
+        def transaction_time_in_thread thread_id, entry, now
+          return entry.elapsed_transaction_time unless in_transaction? thread_id
+
+          # Count the portion of the transaction that's elapsed so far,...
+          elapsed = record_elapsed_transaction_time_until now, thread_id: thread_id
+
+          # ...then readjust the transaction start time to the next harvest
+          split_transaction_at_harvest now, thread_id: thread_id
+
+          elapsed
+        end
+      end
+    end
+  end
+end

data/lib/new_relic/version.rb

@@ -11,7 +11,7 @@ module NewRelic
     end
 
     MAJOR = 5
-    MINOR = 3
+    MINOR = 4
     TINY  = 0
 
     begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: newrelic_rpm
 version: !ruby/object:Gem::Version
-  version: 5.3.0.346
+  version: 5.4.0.347
 platform: ruby
 authors:
 - Matthew Wear
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-26 00:00:00.000000000 Z
+date: 2018-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -202,7 +202,6 @@ files:
 - lib/new_relic/agent/attribute_processing.rb
 - lib/new_relic/agent/audit_logger.rb
 - lib/new_relic/agent/autostart.rb
-- lib/new_relic/agent/busy_calculator.rb
 - lib/new_relic/agent/chained_call.rb
 - lib/new_relic/agent/commands/agent_command.rb
 - lib/new_relic/agent/commands/agent_command_router.rb
@@ -393,6 +392,7 @@ files:
 - lib/new_relic/agent/transaction_metrics.rb
 - lib/new_relic/agent/transaction_sampler.rb
 - lib/new_relic/agent/transaction_state.rb
+- lib/new_relic/agent/transaction_time_aggregator.rb
 - lib/new_relic/agent/transaction_timings.rb
 - lib/new_relic/agent/utilization/aws.rb
 - lib/new_relic/agent/utilization/azure.rb

data/lib/new_relic/agent/busy_calculator.rb

@@ -1,117 +0,0 @@
-# encoding: utf-8
-# This file is distributed under New Relic's license terms.
-# See https://github.com/newrelic/rpm/blob/master/LICENSE for complete details.
-
-require 'new_relic/agent/transaction_state'
-
-module NewRelic
-  module Agent
-    # This module supports calculation of actual time spent processing requests over the course of
-    # one harvest period.  It's similar to what you would get if you just added up all the
-    # execution times of controller calls, however that will be inaccurate when requests
-    # span the minute boundaries.  This module manages accounting of requests not yet
-    # completed.
-    #
-    # Calls are re-entrant.  All start calls must be paired with finish
-    # calls, or a reset call.
-    module BusyCalculator
-
-      extend self
-
-      # For testability, add accessors:
-      attr_reader :harvest_start, :accumulator
-
-      # sets up busy calculations based on the start and end of
-      # transactions - used for a rough estimate of what percentage of
-      # wall clock time is spent processing requests
-      def dispatcher_start(time) #THREAD_LOCAL_ACCESS
-        state = TransactionState.tl_get
-        state.busy_entries ||= 0
-        callers = state.busy_entries += 1
-        return if callers > 1
-        @lock.synchronize do
-          @entrypoint_stack.push time
-        end
-      end
-
-      # called when a transaction finishes, to add time to the
-      # instance variable accumulator. this is harvested when we send
-      # data to the server
-      def dispatcher_finish(end_time = nil) #THREAD_LOCAL_ACCESS
-        state = TransactionState.tl_get
-        # If #dispatcher_start hasn't been called at least once, abort early
-        return unless state.busy_entries
-
-        end_time ||= time_now
-        callers = state.busy_entries -= 1
-
-        # Ignore nested calls
-        return if callers > 0
-
-        @lock.synchronize do
-          if @entrypoint_stack.empty?
-            ::NewRelic::Agent.logger.warn("Stack underflow tracking dispatcher entry and exit!\n  #{caller.join("  \n")}")
-          else
-            @accumulator += (end_time - @entrypoint_stack.pop).to_f
-          end
-        end
-      end
-
-      # this returns the size of the entry point stack, which
-      # determines how many transactions are running
-      def busy_count
-        @entrypoint_stack.size
-      end
-
-      # Reset the state of the information accumulated by all threads,
-      # but only reset the recursion counter for this thread.
-      def reset #THREAD_LOCAL_ACCESS
-        @entrypoint_stack = []
-        TransactionState.tl_get.busy_entries = 0
-        @lock ||= Mutex.new
-        @accumulator = 0
-        @harvest_start = time_now
-      end
-
-
-      # Called before uploading to to the server to collect current busy stats.
-      def harvest_busy
-        busy = 0
-        t0 = time_now
-        @lock.synchronize do
-          busy = accumulator
-          @accumulator = 0
-
-          # Walk through the stack and capture all times up to
-          # now for entrypoints
-          @entrypoint_stack.size.times do |frame|
-            busy += (t0 - @entrypoint_stack[frame]).to_f
-            @entrypoint_stack[frame] = t0
-          end
-
-        end
-
-        busy = 0.0 if busy < 0.0 # don't go below 0%
-
-        time_window = (t0 - harvest_start).to_f
-        time_window = 1.0 if time_window == 0.0  # protect against divide by zero
-
-        busy = busy / time_window
-
-        if Agent.config[:report_instance_busy]
-          NewRelic::Agent.record_metric('Instance/Busy', busy)
-        end
-        @harvest_start = t0
-      end
-
-      private
-
-      # so we can stub Time.now only for the BusyCalculator in tests
-      def time_now
-        Time.now
-      end
-
-      self.reset
-    end
-  end
-end