mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/threading/backtrace_node.rb

@@ -0,0 +1,135 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Threading
+      MAX_THREAD_PROFILE_DEPTH = 500
+      UNKNOWN_LINE_NUMBER = -1
+
+      class BacktraceBase
+        attr_reader :children
+
+        def initialize
+          @children = []
+          @depth    = 0
+        end
+
+        def add_child_unless_present(child)
+          child.depth = @depth + 1
+          @children << child unless @children.include? child
+        end
+
+        def add_child(child)
+          child.depth = @depth + 1
+          @children << child
+        end
+
+        def find_child(raw_line)
+          @children.find { |child| child.raw_line == raw_line }
+        end
+      end
+
+
+      class BacktraceRoot < BacktraceBase
+        attr_reader :flattened
+
+        def initialize
+          super
+          @flattened = []
+        end
+
+        def ==(other)
+          true # all roots are at the same depth and have no raw_line
+        end
+
+        def as_array
+          @children.map { |c| c.as_array }.compact
+        end
+
+        def aggregate(backtrace)
+          current = self
+
+          depth = 0
+          backtrace.reverse_each do |frame|
+            break if depth >= MAX_THREAD_PROFILE_DEPTH
+
+            existing_node = current.find_child(frame)
+            if existing_node
+              node = existing_node
+            else
+              node = Threading::BacktraceNode.new(frame)
+              current.add_child(node)
+              @flattened << node
+            end
+
+            node.runnable_count += 1
+            current = node
+            depth += 1
+          end
+        end
+
+        def dump_string
+          result = "#<BacktraceRoot:#{object_id}>"
+          child_results = @children.map { |c| c.dump_string(2) }.join("\n")
+          result << "\n" unless child_results.empty?
+          result << child_results
+        end
+      end
+
+
+      class BacktraceNode < BacktraceBase
+        attr_reader   :file, :method, :line_no, :raw_line, :as_array
+        attr_accessor :runnable_count, :depth
+
+        def initialize(line)
+          super()
+          @raw_line = line
+          @children = []
+          @runnable_count = 0
+        end
+
+        def ==(other)
+          (
+            @raw_line       == other.raw_line &&
+            @depth          == other.depth    &&
+            @runnable_count == other.runnable_count
+          )
+        end
+
+        def mark_for_array_conversion
+          @as_array = []
+        end
+
+        include Mmtrix::Coerce
+
+        def complete_array_conversion
+          child_arrays = @children.map { |c| c.as_array }.compact
+
+          file, method, line = parse_backtrace_frame(@raw_line)
+
+          @as_array << [string(file), string(method), line ? int(line) : UNKNOWN_LINE_NUMBER]
+          @as_array << int(@runnable_count)
+          @as_array << 0
+          @as_array << child_arrays
+        end
+
+        def dump_string(indent=0)
+          @file, @method, @line_no = parse_backtrace_frame(@raw_line)
+          result = "#{" " * indent}#<BacktraceNode:#{object_id} [#{@runnable_count}] #{@file}:#{@line_no} in #{@method}>"
+          child_results = @children.map { |c| c.dump_string(indent+2) }.join("\n")
+          result << "\n" unless child_results.empty?
+          result << child_results
+        end
+
+        # Returns [filename, method, line number]
+        def parse_backtrace_frame(frame)
+          frame =~ /([^:]*)(\:(\d+))?\:in `(.*)'/
+          [$1, $4, $3] # sic
+        end
+      end
+
+    end
+  end
+end

data/lib/mmtrix/agent/threading/backtrace_service.rb

@@ -0,0 +1,283 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Threading
+      class BacktraceService
+        ALL_TRANSACTIONS = "**ALL**".freeze
+
+        def self.is_supported?
+          RUBY_VERSION >= "1.9.2" && !is_resque?
+        end
+
+        # Because of Resque's forking, we don't poll thread backtraces for it.
+        # To accomplish that would require starting a new backtracing thread in
+        # each forked worker, and merging profiles across the pipe channel.
+        def self.is_resque?
+          Mmtrix::Agent.config[:dispatcher] == :resque
+        end
+
+        attr_reader :worker_loop, :buffer, :effective_polling_period,
+                    :overhead_percent_threshold
+        attr_accessor :worker_thread, :profile_agent_code
+
+        def initialize(event_listener=nil)
+          @profiles = {}
+          @buffer = {}
+
+          # synchronizes access to @profiles and @buffer above
+          @lock = Mutex.new
+
+          @running = false
+          @profile_agent_code = false
+          @worker_loop = Mmtrix::Agent::WorkerLoop.new
+
+          # Memoize overhead % to avoid getting stale OR looked up every poll
+          @overhead_percent_threshold = Mmtrix::Agent.config[:'xray_session.max_profile_overhead']
+          Mmtrix::Agent.config.register_callback(:'xray_session.max_profile_overhead') do |new_value|
+            @overhead_percent_threshold = new_value
+          end
+
+          if event_listener
+            event_listener.subscribe(:transaction_finished, &method(:on_transaction_finished))
+          end
+        end
+
+        # Public interface
+
+        def running?
+          @running
+        end
+
+        def subscribe(transaction_name, command_arguments={})
+          if self.class.is_resque?
+            Mmtrix::Agent.logger.info("Backtracing threads on Resque is not supported, so not subscribing transaction '#{transaction_name}'")
+            return
+          end
+
+          if !self.class.is_supported?
+            Mmtrix::Agent.logger.debug("Backtracing not supported, so not subscribing transaction '#{transaction_name}'")
+            return
+          end
+
+          Mmtrix::Agent.logger.debug("Backtrace Service subscribing transaction '#{transaction_name}'")
+
+          profile = ThreadProfile.new(command_arguments)
+
+          @lock.synchronize do
+            @profiles[transaction_name] = profile
+            update_values_from_profiles
+          end
+
+          start
+          profile
+        end
+
+        def unsubscribe(transaction_name)
+          return unless self.class.is_supported?
+
+          Mmtrix::Agent.logger.debug("Backtrace Service unsubscribing transaction '#{transaction_name}'")
+          @lock.synchronize do
+            @profiles.delete(transaction_name)
+            if @profiles.empty?
+              stop
+            else
+              update_values_from_profiles
+            end
+          end
+        end
+
+        def update_values_from_profiles
+          self.effective_polling_period = find_effective_polling_period
+          self.profile_agent_code = should_profile_agent_code?
+        end
+
+        def subscribed?(transaction_name)
+          @lock.synchronize do
+            @profiles.has_key?(transaction_name)
+          end
+        end
+
+        def harvest(transaction_name)
+          @lock.synchronize do
+            if @profiles[transaction_name]
+              profile = @profiles.delete(transaction_name)
+              profile.finished_at = Time.now
+              @profiles[transaction_name] = ThreadProfile.new(profile.command_arguments)
+              profile
+            end
+          end
+        end
+
+        def on_transaction_finished(payload)
+          name     = payload[:name]
+          start    = payload[:start_timestamp]
+          duration = payload[:duration]
+          thread   = payload[:thread] || Thread.current
+          bucket   = payload[:bucket]
+          @lock.synchronize do
+            backtraces = @buffer.delete(thread)
+            if backtraces && @profiles.has_key?(name)
+              aggregate_backtraces(backtraces, name, start, duration, bucket, thread)
+            end
+          end
+        end
+
+        # Internals
+
+        # This method is expected to be called with @lock held.
+        def aggregate_backtraces(backtraces, name, start, duration, bucket, thread)
+          end_time = start + duration
+          backtraces.each do |(timestamp, backtrace)|
+            if timestamp >= start && timestamp < end_time
+              @profiles[name].aggregate(backtrace, bucket, thread)
+            end
+          end
+        end
+
+        def start
+          return if @running || !self.class.is_supported?
+
+          @running = true
+          self.worker_thread = AgentThread.create('Backtrace Service') do
+            # Not passing period because we expect it's already been set.
+            self.worker_loop.run(&method(:poll))
+          end
+        end
+
+        # This method is expected to be called with @lock held
+        def stop
+          return unless @running
+          @running = false
+          self.worker_loop.stop
+
+          @buffer = {}
+        end
+
+        def effective_polling_period=(new_period)
+          @effective_polling_period = new_period
+          self.worker_loop.period = new_period
+        end
+
+        def poll
+          poll_start = Time.now
+
+          @lock.synchronize do
+            AgentThread.list.each do |thread|
+              sample_thread(thread)
+            end
+            @profiles.each_value { |p| p.increment_poll_count }
+            @buffer.delete_if { |thread, _| !thread.alive? }
+          end
+
+          end_time = Time.now
+          adjust_polling_time(end_time, poll_start)
+          record_supportability_metrics(end_time, poll_start)
+        end
+
+        # This method is expected to be called with @lock held.
+        attr_reader :profiles
+
+        # This method is expected to be called with @lock held.
+        def should_buffer?(bucket)
+          allowed_bucket?(bucket) && watching_for_transaction?
+        end
+
+        # This method is expected to be called with @lock held.
+        def need_backtrace?(bucket)
+          (
+            bucket != :ignore &&
+            (@profiles[ALL_TRANSACTIONS] || should_buffer?(bucket))
+          )
+        end
+
+        # This method is expected to be called with @lock held
+        def watching_for_transaction?
+          @profiles.size > 1 ||
+          (@profiles.size == 1 && @profiles[ALL_TRANSACTIONS].nil?)
+        end
+
+        def allowed_bucket?(bucket)
+          bucket == :request || bucket == :background
+        end
+
+        MAX_BUFFER_LENGTH = 500
+
+        # This method is expected to be called with @lock held.
+        def buffer_backtrace_for_thread(thread, timestamp, backtrace, bucket)
+          if should_buffer?(bucket)
+            @buffer[thread] ||= []
+            if @buffer[thread].length < MAX_BUFFER_LENGTH
+              @buffer[thread] << [timestamp, backtrace]
+            else
+              Mmtrix::Agent.increment_metric('Supportability/XraySessions/DroppedBacktraces')
+            end
+          end
+        end
+
+        # This method is expected to be called with @lock held.
+        def aggregate_global_backtrace(backtrace, bucket, thread)
+          if @profiles[ALL_TRANSACTIONS]
+            @profiles[ALL_TRANSACTIONS].aggregate(backtrace, bucket, thread)
+          end
+        end
+
+        # This method is expected to be called with @lock held.
+        def sample_thread(thread)
+          bucket = AgentThread.bucket_thread(thread, @profile_agent_code)
+
+          if need_backtrace?(bucket)
+            timestamp = Time.now.to_f
+            backtrace = AgentThread.scrub_backtrace(thread, @profile_agent_code)
+            aggregate_global_backtrace(backtrace, bucket, thread)
+            buffer_backtrace_for_thread(thread, timestamp, backtrace, bucket)
+          end
+        end
+
+        # This method is expected to be called with @lock held.
+        def find_effective_polling_period
+          @profiles.values.map { |p| p.requested_period }.min
+        end
+
+        # This method is expected to be called with @lock held.
+        def should_profile_agent_code?
+          @profiles.values.any? { |p| p.profile_agent_code }
+        end
+
+        # If our overhead % exceeds the threshold, bump the next poll period
+        # relative to how much larger our overhead is than allowed
+        def adjust_polling_time(now, poll_start)
+          duration = now - poll_start
+          overhead_percent = duration / effective_polling_period
+
+          if overhead_percent > self.overhead_percent_threshold
+            scale_up_by = overhead_percent / self.overhead_percent_threshold
+            worker_loop.period = effective_polling_period * scale_up_by
+          else
+            worker_loop.period = effective_polling_period
+          end
+        end
+
+        def record_supportability_metrics(now, poll_start)
+          record_polling_time(now, poll_start)
+          record_skew(poll_start)
+        end
+
+        def record_polling_time(now, poll_start)
+          Mmtrix::Agent.record_metric('Supportability/ThreadProfiler/PollingTime', now - poll_start)
+        end
+
+        def record_skew(poll_start)
+          if @last_poll
+            skew = poll_start - @last_poll - worker_loop.period
+            Mmtrix::Agent.record_metric('Supportability/ThreadProfiler/Skew', skew)
+          end
+          @last_poll = poll_start
+        end
+
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/threading/thread_profile.rb

@@ -0,0 +1,149 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'set'
+require 'mmtrix/agent/worker_loop'
+require 'mmtrix/agent/threading/backtrace_node'
+
+# Data structure for representing a thread profile
+
+module Mmtrix
+  module Agent
+    module Threading
+
+      class ThreadProfile
+
+        attr_reader :profile_id, :traces, :sample_period,
+          :duration, :poll_count, :backtrace_count, :failure_count,
+          :created_at, :xray_id, :command_arguments, :profile_agent_code
+        attr_accessor :finished_at
+
+        def initialize(command_arguments={})
+          @command_arguments  = command_arguments
+          @profile_id         = command_arguments.fetch('profile_id', -1)
+          @duration           = command_arguments.fetch('duration', 120)
+          @sample_period      = command_arguments.fetch('sample_period', 0.1)
+          @profile_agent_code = command_arguments.fetch('profile_agent_code', false)
+          @xray_id            = command_arguments.fetch('x_ray_id', nil)
+          @finished = false
+
+          @traces = {
+            :agent      => BacktraceRoot.new,
+            :background => BacktraceRoot.new,
+            :other      => BacktraceRoot.new,
+            :request    => BacktraceRoot.new
+          }
+
+          @poll_count = 0
+          @backtrace_count = 0
+          @failure_count = 0
+          @unique_threads = []
+
+          @created_at = Time.now
+        end
+
+        def requested_period
+          @sample_period
+        end
+
+        def increment_poll_count
+          @poll_count += 1
+        end
+
+        def sample_count
+          xray? ? @backtrace_count : @poll_count
+        end
+
+        def xray?
+          !!@xray_id
+        end
+
+        def empty?
+          @backtrace_count == 0
+        end
+
+        def unique_thread_count
+          return 0 if @unique_threads.nil?
+          @unique_threads.length
+        end
+
+        def aggregate(backtrace, bucket, thread)
+          if backtrace.nil?
+            @failure_count += 1
+          else
+            @backtrace_count += 1
+            @traces[bucket].aggregate(backtrace)
+            @unique_threads << thread unless @unique_threads.include?(thread)
+          end
+        end
+
+        def convert_N_trace_nodes_to_arrays(count_to_keep) #THREAD_LOCAL_ACCESS
+          all_nodes = @traces.values.map { |n| n.flattened }.flatten
+
+          Mmtrix::Agent.instance.stats_engine.
+            tl_record_supportability_metric_count("ThreadProfiler/NodeCount", all_nodes.size)
+
+          all_nodes.sort! do |a, b|
+            # we primarily prefer higher runnable_count
+            comparison = b.runnable_count <=> a.runnable_count
+            # we secondarily prefer lower depth
+            comparison = a.depth          <=> b.depth if comparison == 0
+            # it is thus impossible for any child to preceed their parent
+            comparison
+          end
+
+          all_nodes.each_with_index do |n, i|
+            break if i >= count_to_keep
+            n.mark_for_array_conversion
+          end
+          all_nodes.each_with_index do |n, i|
+            break if i >= count_to_keep
+            n.complete_array_conversion
+          end
+        end
+
+        THREAD_PROFILER_NODES = 20_000
+
+        include Mmtrix::Coerce
+
+        def generate_traces
+          convert_N_trace_nodes_to_arrays(THREAD_PROFILER_NODES)
+
+          {
+            "OTHER"      => @traces[:other     ].as_array,
+            "REQUEST"    => @traces[:request   ].as_array,
+            "AGENT"      => @traces[:agent     ].as_array,
+            "BACKGROUND" => @traces[:background].as_array
+          }
+        end
+
+        def to_collector_array(encoder)
+          encoded_trace_tree = encoder.encode(generate_traces, :skip_normalization => true)
+          result = [
+            int(self.profile_id),
+            float(self.created_at),
+            float(self.finished_at),
+            int(self.sample_count),
+            encoded_trace_tree,
+            int(self.unique_thread_count),
+            0 # runnable thread count, which we don't track
+          ]
+          result << int(@xray_id) if xray?
+          result
+        end
+
+        def to_log_description
+          id = if xray?
+                 "@xray_id: #{xray_id}"
+               else
+                 "@profile_id: #{profile_id}"
+               end
+
+          "#<ThreadProfile:#{object_id} #{id} @command_arguments=#{@command_arguments.inspect}>"
+        end
+
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/traced_method_stack.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class TracedMethodFrame
+      attr_reader :tag
+      attr_accessor :name, :start_time, :children_time
+      def initialize(tag, start_time)
+        @tag = tag
+        @start_time = start_time
+        @children_time = 0
+      end
+    end
+
+    # TracedMethodStack is responsible for tracking the push and pop of methods
+    # that we are tracing, notifying the transaction sampler, and calculating
+    # exclusive time when a method is complete. This is allowed whether a
+    # transaction is in progress not.
+    class TracedMethodStack
+      def initialize
+        @stack = []
+      end
+
+      # Pushes a frame onto the transaction stack - this generates a
+      # Agent::Transaction::TraceNode at the end of transaction execution.
+      #
+      # The generated node won't be named until pop_frame is called.
+      #
+      # +tag+ should be a Symbol, and is only for debugging purposes to
+      # identify this frame if the stack gets corrupted.
+      def push_frame(state, tag, time = Time.now.to_f)
+        transaction_sampler.notice_push_frame(state, time) if sampler_enabled?
+        frame = TracedMethodFrame.new(tag, time)
+        @stack.push frame
+        frame
+      end
+
+      # Pops a frame off the transaction stack - this updates the transaction
+      # sampler that we've finished execution of a traced method.
+      #
+      # +expected_frame+ should be TracedMethodFrame from the corresponding
+      # push_frame call.
+      #
+      # +name+ will be applied to the generated transaction trace node.
+      def pop_frame(state, expected_frame, name, time, deduct_call_time_from_parent=true)
+        frame = fetch_matching_frame(expected_frame)
+
+        note_children_time(frame, time, deduct_call_time_from_parent)
+
+        transaction_sampler.notice_pop_frame(state, name, time) if sampler_enabled?
+        frame.name = name
+        frame
+      end
+
+      def fetch_matching_frame(expected_frame)
+        while frame = @stack.pop
+          if frame == expected_frame
+            return frame
+          else
+            Mmtrix::Agent.logger.info("Unexpected frame in traced method stack: #{frame.inspect} expected to be #{expected_frame.inspect}")
+            Mmtrix::Agent.logger.debug do
+              ["Backtrace for unexpected frame: ", caller.join("\n")]
+            end
+          end
+        end
+
+        raise "Frame not found in blame stack: #{expected_frame.inspect}"
+      end
+
+      def note_children_time(frame, time, deduct_call_time_from_parent)
+        if !@stack.empty?
+          if deduct_call_time_from_parent
+            @stack.last.children_time += (time - frame.start_time)
+          else
+            @stack.last.children_time += frame.children_time
+          end
+        end
+      end
+
+      def sampler_enabled?
+        Agent.config[:'transaction_tracer.enabled'] || Agent.config[:developer_mode]
+      end
+
+      def transaction_sampler
+        Agent.instance.transaction_sampler
+      end
+
+      def clear
+        @stack.clear
+      end
+
+      def empty?
+        @stack.empty?
+      end
+    end
+  end
+end