mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/aws_info.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    class AWSInfo
+
+      class ResponseError < StandardError; end
+
+      attr_reader :instance_type, :instance_id, :availability_zone
+
+      def initialize
+        handle_remote_calls do
+          @instance_type = remote_fetch('instance-type')
+          @instance_id = remote_fetch('instance-id')
+          @availability_zone = remote_fetch('placement/availability-zone')
+        end
+      end
+
+      def loaded?
+        instance_type && instance_id && availability_zone
+      end
+
+      def to_collector_hash
+        {
+          :id => instance_id,
+          :type => instance_type,
+          :zone => availability_zone
+        }
+      end
+
+      private
+
+      def reset
+        @instance_type = @instance_id = @availability_zone = nil
+      end
+
+      REMOTE_DATA_VALID_CHARS = /^[0-9a-zA-Z_ .\/-]$/.freeze
+      INSTANCE_HOST = '169.254.169.254'
+      API_VERSION   = '2008-02-01'
+
+      def remote_fetch(remote_key)
+        uri = URI("http://#{INSTANCE_HOST}/#{API_VERSION}/meta-data/#{remote_key}")
+        response = Net::HTTP::get(uri)
+
+        data = filter_remote_data(response)
+
+        unless data
+          Mmtrix::Agent.increment_metric('Supportability/utilization/aws/error')
+          raise ResponseError, "Fetching instance metadata for #{remote_key.inspect} returned invalid data: #{response.inspect}"
+        end
+
+        data
+      end
+
+      def handle_remote_calls
+        begin
+          Timeout::timeout(1) do
+            yield
+          end
+        rescue Timeout::Error
+          handle_error "UtilizationData timed out fetching remote keys."
+        rescue StandardError => e
+          handle_error "UtilizationData encountered error fetching remote keys", e
+        end
+      end
+
+      def handle_error(*messages)
+        Mmtrix::Agent.logger.debug(*messages)
+        reset
+      end
+
+      def filter_remote_data(data_str)
+        return nil unless data_str.kind_of?(String)
+        return nil unless data_str.bytesize <= 255
+
+        data_str.each_char do |ch|
+          next if ch =~ REMOTE_DATA_VALID_CHARS
+          code_point = ch[0].ord # this works in Ruby 1.8.7 - 2.1.2
+          next if code_point >= 0x80
+
+          return nil # it's in neither set of valid characters
+        end
+
+        data_str
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/busy_calculator.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/transaction_state'
+
+module Mmtrix
+  module Agent
+    # This module supports calculation of actual time spent processing requests over the course of
+    # one harvest period.  It's similar to what you would get if you just added up all the
+    # execution times of controller calls, however that will be inaccurate when requests
+    # span the minute boundaries.  This module manages accounting of requests not yet
+    # completed.
+    #
+    # Calls are re-entrant.  All start calls must be paired with finish
+    # calls, or a reset call.
+    module BusyCalculator
+
+      extend self
+
+      # For testability, add accessors:
+      attr_reader :harvest_start, :accumulator
+
+      # sets up busy calculations based on the start and end of
+      # transactions - used for a rough estimate of what percentage of
+      # wall clock time is spent processing requests
+      def dispatcher_start(time) #THREAD_LOCAL_ACCESS
+        state = TransactionState.tl_get
+        state.busy_entries ||= 0
+        callers = state.busy_entries += 1
+        return if callers > 1
+        @lock.synchronize do
+          @entrypoint_stack.push time
+        end
+      end
+
+      # called when a transaction finishes, to add time to the
+      # instance variable accumulator. this is harvested when we send
+      # data to the server
+      def dispatcher_finish(end_time = nil) #THREAD_LOCAL_ACCESS
+        state = TransactionState.tl_get
+        # If #dispatcher_start hasn't been called at least once, abort early
+        return unless state.busy_entries
+
+        end_time ||= time_now
+        callers = state.busy_entries -= 1
+
+        # Ignore nested calls
+        return if callers > 0
+
+        @lock.synchronize do
+          if @entrypoint_stack.empty?
+            ::Mmtrix::Agent.logger.warn("Stack underflow tracking dispatcher entry and exit!\n  #{caller.join("  \n")}")
+          else
+            @accumulator += (end_time - @entrypoint_stack.pop).to_f
+          end
+        end
+      end
+
+      # this returns the size of the entry point stack, which
+      # determines how many transactions are running
+      def busy_count
+        @entrypoint_stack.size
+      end
+
+      # Reset the state of the information accumulated by all threads,
+      # but only reset the recursion counter for this thread.
+      def reset #THREAD_LOCAL_ACCESS
+        @entrypoint_stack = []
+        TransactionState.tl_get.busy_entries = 0
+        @lock ||= Mutex.new
+        @accumulator = 0
+        @harvest_start = time_now
+      end
+
+
+      # Called before uploading to to the server to collect current busy stats.
+      def harvest_busy
+        busy = 0
+        t0 = time_now
+        @lock.synchronize do
+          busy = accumulator
+          @accumulator = 0
+
+          # Walk through the stack and capture all times up to
+          # now for entrypoints
+          @entrypoint_stack.size.times do |frame|
+            busy += (t0 - @entrypoint_stack[frame]).to_f
+            @entrypoint_stack[frame] = t0
+          end
+
+        end
+
+        busy = 0.0 if busy < 0.0 # don't go below 0%
+
+        time_window = (t0 - harvest_start).to_f
+        time_window = 1.0 if time_window == 0.0  # protect against divide by zero
+
+        busy = busy / time_window
+
+        if Agent.config[:report_instance_busy]
+          Mmtrix::Agent.record_metric('Instance/Busy', busy)
+        end
+        @harvest_start = t0
+      end
+
+      private
+
+      # so we can stub Time.now only for the BusyCalculator in tests
+      def time_now
+        Time.now
+      end
+
+      self.reset
+    end
+  end
+end

data/lib/mmtrix/agent/chained_call.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This class is used by Mmtrix::Agent.set_sql_obfuscator to chain multiple
+# obfuscation blocks when not using the default :replace action
+class Mmtrix::ChainedCall
+  def initialize(block1, block2)
+    @block1 = block1
+    @block2 = block2
+  end
+
+  def call(sql)
+    sql = @block1.call(sql)
+    @block2.call(sql)
+  end
+end

data/lib/mmtrix/agent/commands/agent_command.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Commands
+      class AgentCommand
+        attr_reader :id, :name, :arguments
+
+        def initialize(raw_collector_command)
+          @id        = raw_collector_command[0]
+          @name      = raw_collector_command[1]["name"]
+          @arguments = raw_collector_command[1]["arguments"]
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/commands/agent_command_router.rb

@@ -0,0 +1,160 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This class is the central point for dispatching get_agent_commands messages
+# to the various components that actually process them.
+#
+# This could be evented further, but we eventually need direct access to things
+# like the ThreadProfiler, so it's simpler to just keep it together here.
+
+require 'mmtrix/agent/commands/agent_command'
+require 'mmtrix/agent/commands/xray_session_collection'
+require 'mmtrix/agent/threading/backtrace_service'
+
+module Mmtrix
+  module Agent
+    module Commands
+      class AgentCommandRouter
+        attr_reader :handlers
+
+        attr_accessor :thread_profiler_session, :backtrace_service,
+                      :xray_session_collection
+
+        def initialize(event_listener=nil)
+          @handlers    = Hash.new { |*| Proc.new { |cmd| self.unrecognized_agent_command(cmd) } }
+
+          @backtrace_service = Threading::BacktraceService.new(event_listener)
+
+          @thread_profiler_session = ThreadProfilerSession.new(@backtrace_service)
+          @xray_session_collection = XraySessionCollection.new(@backtrace_service, event_listener)
+
+          @handlers['start_profiler'] = Proc.new { |cmd| thread_profiler_session.handle_start_command(cmd) }
+          @handlers['stop_profiler']  = Proc.new { |cmd| thread_profiler_session.handle_stop_command(cmd) }
+          @handlers['active_xray_sessions'] = Proc.new { |cmd| xray_session_collection.handle_active_xray_sessions(cmd) }
+
+          if event_listener
+            event_listener.subscribe(:before_shutdown, &method(:on_before_shutdown))
+          end
+        end
+
+        def mmtrix_service
+          Mmtrix::Agent.instance.service
+        end
+
+        def check_for_and_handle_agent_commands
+          commands = get_agent_commands
+
+          stop_xray_sessions unless active_xray_command?(commands)
+
+          results = invoke_commands(commands)
+          mmtrix_service.agent_command_results(results) unless results.empty?
+        end
+
+        def stop_xray_sessions
+          self.xray_session_collection.stop_all_sessions
+        end
+
+        def active_xray_command?(commands)
+          commands.any? {|command| command.name == 'active_xray_sessions'}
+        end
+
+        def on_before_shutdown(*args)
+          if self.thread_profiler_session.running?
+            self.thread_profiler_session.stop(true)
+          end
+        end
+
+        def harvest!
+          profiles = []
+          profiles += harvest_from_xray_session_collection
+          profiles += harvest_from_thread_profiler_session
+          log_profiles(profiles)
+          profiles
+        end
+
+        # We don't currently support merging thread profiles that failed to send
+        # back into the AgentCommandRouter, so we just no-op this method.
+        # Same with reset! - we don't support asynchronous cancellation of a
+        # running thread profile or X-Ray session currently.
+        def merge!(*args); end
+        def reset!; end
+
+        def harvest_from_xray_session_collection
+          self.xray_session_collection.harvest_thread_profiles
+        end
+
+        def harvest_from_thread_profiler_session
+          if self.thread_profiler_session.ready_to_harvest?
+            self.thread_profiler_session.stop(true)
+            [self.thread_profiler_session.harvest]
+          else
+            []
+          end
+        end
+
+        def log_profiles(profiles)
+          if profiles.empty?
+            ::Mmtrix::Agent.logger.debug "No thread profiles with data found to send."
+          else
+            profile_descriptions = profiles.map { |p| p.to_log_description }
+            ::Mmtrix::Agent.logger.debug "Sending thread profiles [#{profile_descriptions.join(", ")}]"
+          end
+        end
+
+        def get_agent_commands
+          commands = mmtrix_service.get_agent_commands
+          Mmtrix::Agent.logger.debug "Received get_agent_commands = #{commands.inspect}"
+          commands.map {|collector_command| AgentCommand.new(collector_command)}
+        end
+
+        def invoke_commands(agent_commands)
+          results = {}
+
+          agent_commands.each do |agent_command|
+            results[agent_command.id.to_s] = invoke_command(agent_command)
+          end
+
+          results
+        end
+
+        class AgentCommandError < StandardError
+        end
+
+        def invoke_command(agent_command)
+          begin
+            call_handler_for(agent_command)
+            return success
+          rescue AgentCommandError => e
+            Mmtrix::Agent.logger.debug(e)
+            error(e)
+          end
+        end
+
+        SUCCESS_RESULT = {}.freeze
+        ERROR_KEY = "error"
+
+        def success
+          SUCCESS_RESULT
+        end
+
+        def error(err)
+          { ERROR_KEY => err.message }
+        end
+
+        def call_handler_for(agent_command)
+          handler = select_handler(agent_command)
+          handler.call(agent_command)
+        end
+
+        def select_handler(agent_command)
+          @handlers[agent_command.name]
+        end
+
+        def unrecognized_agent_command(agent_command)
+          Mmtrix::Agent.logger.debug("Unrecognized agent command #{agent_command.inspect}")
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/commands/thread_profiler_session.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/threading/agent_thread'
+require 'mmtrix/agent/threading/backtrace_service'
+require 'mmtrix/agent/threading/thread_profile'
+
+module Mmtrix
+  module Agent
+    module Commands
+
+      class ThreadProfilerSession
+
+        def initialize(backtrace_service)
+          @backtrace_service = backtrace_service
+        end
+
+        def handle_start_command(agent_command)
+          raise_unsupported_error unless Mmtrix::Agent::Threading::BacktraceService.is_supported?
+          raise_thread_profiler_disabled unless enabled?
+          raise_already_started_error if running?
+          start(agent_command)
+        end
+
+        def handle_stop_command(agent_command)
+          report_data = agent_command.arguments.fetch("report_data", true)
+          stop(report_data)
+        end
+
+        def start(agent_command)
+          Mmtrix::Agent.logger.debug("Starting Thread Profiler.")
+          profile = @backtrace_service.subscribe(
+            Mmtrix::Agent::Threading::BacktraceService::ALL_TRANSACTIONS,
+            agent_command.arguments
+          )
+
+          @started_at = Time.now
+          @duration = profile.duration if profile
+        end
+
+        def stop(report_data)
+          return unless running?
+          Mmtrix::Agent.logger.debug("Stopping Thread Profiler.")
+          @finished_profile = @backtrace_service.harvest(Mmtrix::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+          @backtrace_service.unsubscribe(Mmtrix::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+          @finished_profile = nil if !report_data
+        end
+
+        def harvest
+          Mmtrix::Agent.logger.debug("Harvesting from Thread Profiler #{@finished_profile.to_log_description unless @finished_profile.nil?}")
+          profile = @finished_profile
+          @backtrace_service.profile_agent_code = false
+          @finished_profile = nil
+          @started_at = nil
+          profile
+        end
+
+        def enabled?
+          Mmtrix::Agent.config[:'thread_profiler.enabled']
+        end
+
+        def running?
+          @backtrace_service.subscribed?(Mmtrix::Agent::Threading::BacktraceService::ALL_TRANSACTIONS)
+        end
+
+        def ready_to_harvest?
+          past_time? || stopped?
+        end
+
+        def past_time?
+          @started_at && (Time.now > @started_at + @duration)
+        end
+
+        def stopped?
+          !!@finished_profile
+        end
+
+        private
+
+        def raise_command_error(msg)
+          raise Mmtrix::Agent::Commands::AgentCommandRouter::AgentCommandError.new(msg)
+        end
+
+        def raise_unsupported_error
+          msg = <<-EOF
+Thread profiling is only supported on 1.9.2 and greater versions of Ruby.
+We detected running agents capable of profiling, but the profile started with
+an agent running Ruby #{RUBY_VERSION}.
+
+Profiling again might select an appropriate agent, but we recommend running a
+consistent version of Ruby across your application for better results.
+          EOF
+          raise_command_error(msg)
+        end
+
+        def raise_thread_profiler_disabled
+          msg = "Not starting Thread Profiler because of config 'thread_profiler.enabled' = #{enabled?}"
+          raise_command_error(msg)
+        end
+
+        def raise_already_started_error
+          msg = "Profile already in progress. Ignoring agent command to start another."
+          raise_command_error(msg)
+        end
+
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/commands/xray_session.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'forwardable'
+
+module Mmtrix
+  module Agent
+    module Commands
+      class XraySession
+        extend Forwardable
+
+        attr_reader :id, :command_arguments
+        attr_reader :xray_session_name, :key_transaction_name,
+                    :requested_trace_count, :duration, :sample_period
+
+        def initialize(command_arguments)
+          @command_arguments     = command_arguments
+          @id                    = command_arguments.fetch("x_ray_id", nil)
+          @xray_session_name     = command_arguments.fetch("xray_session_name", "")
+          @key_transaction_name  = command_arguments.fetch("key_transaction_name", "")
+          @requested_trace_count = command_arguments.fetch("requested_trace_count", 100)
+          @duration              = command_arguments.fetch("duration", 86400)
+          @sample_period         = command_arguments.fetch("sample_period", 0.1)
+          @run_profiler          = command_arguments.fetch("run_profiler", true)
+        end
+
+        def active?
+          @active
+        end
+
+        def run_profiler?
+          @run_profiler && Mmtrix::Agent.config[:'xray_session.allow_profiles']
+        end
+
+        def activate
+          @active = true
+          @start_time = Time.now
+        end
+
+        def deactivate
+          @active = false
+        end
+
+        def requested_period
+          @sample_period
+        end
+
+        def finished?
+          @start_time + @duration < Time.now
+        end
+      end
+    end
+  end
+end