mmtrix_rpm 1.0.0.1

data/lib/mmtrix/agent/supported_versions.rb

@@ -0,0 +1,259 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    SUPPORTED_VERSIONS =
+    {
+      # Rubies
+      :mri =>
+      {
+        :type        => :ruby,
+        :name        => "MRI",
+        :supported   => ["1.8.7", "1.9.2", "1.9.3", "2.0.0", "~> 2.1.0", "~> 2.2.0" ],
+        :deprecated  => ["1.8.6"],
+        :url         => "https://www.ruby-lang.org",
+        :feed        => "https://www.ruby-lang.org/en/feeds/news.rss",
+        :notes       => [
+          "1.8.7 includes support for Ruby Enterprise Edition (REE)",
+          "Last supported agent on 1.8.6 was 3.6.8.168"]
+      },
+      :jruby =>
+      {
+        :type        => :ruby,
+        :name        => "JRuby",
+        :supported   => ["~> 1.6.0", "~> 1.7.0"],
+        :url         => "http://jruby.org",
+        :feed        => "http://jruby.org/atom.xml"
+      },
+      :rbx =>
+      {
+        :type        => :ruby,
+        :name        => "Rubinius",
+        :supported   => ["~> 2.2.1"],
+        :url         => "http://rubini.us",
+        :feed        => "http://rubini.us/feed/atom.xml"
+      },
+
+      # App servers
+      :passenger =>
+      {
+        :type        => :app_server,
+        :supported   => ["~>2.2", "~>3.0", "~>4.0"],
+        :url         => "http://www.phusionpassenger.com/",
+        :feed        => "http://rubygems.org/gems/passenger/versions.atom"
+      },
+      :thin =>
+      {
+        :type        => :app_server,
+        :supported   => ["~>1.0"],
+        :url         => "http://code.macournoyer.com/thin/",
+        :feed        => "http://rubygems.org/gems/thin/versions.atom"
+      },
+      :unicorn =>
+      {
+        :type        => :app_server,
+        :supported   => ["~>4.0"],
+        :deprecated  => ["~>1.0", "~>2.0", "~>3.0"],
+        :url         => "http://unicorn.bogomips.org/",
+        :feed        => "http://rubygems.org/gems/unicorn/versions.atom"
+      },
+      :puma =>
+      {
+        :type        => :app_server,
+        :supported   => ["~>2.0"],
+        :deprecated  => ["~>1.0"],
+        :url         => "http://puma.io/",
+        :feed        => "http://rubygems.org/gems/puma/versions.atom"
+      },
+      :rainbows =>
+      {
+        :type        => :app_server,
+        :name        => "rainbows!",
+        :experimental=> ["4.5.0"],
+        :url         => "http://rainbows.rubyforge.org/",
+        :feed        => "http://rubygems.org/gems/rainbows/versions.atom"
+      },
+      :webrick =>
+      {
+        :type        => :app_server,
+        :notes       => [ "Supported for all agent-supported versions of Ruby" ]
+      },
+
+      # Web frameworks
+      :rails =>
+      {
+        :type        => :web,
+        :supported   => ["~>2.1.0", "~>2.2.0", "~>2.3.0", "~3.0.0", "~>3.1.0", "~>3.2.0", "~>4.0.0", "~>4.1.0", "~>4.2.0"],
+        :deprecated  => ["~>2.0.0"],
+        :url         => "https://rubygems.org/gems/rails",
+        :feed        => "https://rubygems.org/gems/rails/versions.atom",
+        :notes       => ["Last supported agent for 2.0.x was 3.6.8.168"]
+      },
+      :sinatra =>
+      {
+        :type        => :web,
+        :supported   => ["~>1.2.0", "~>1.3.0", "~>1.4.0"],
+        :url         => "https://rubygems.org/gems/sinatra",
+        :feed        => "https://rubygems.org/gems/sinatra/versions.atom"
+      },
+      :padrino =>
+      {
+        :type        => :web,
+        :supported   => ["~>0.10"],
+        :url         => "https://rubygems.org/gems/padrino",
+        :feed        => "https://rubygems.org/gems/padrino/versions.atom"
+      },
+      :rack =>
+      {
+        :type        => :web,
+        :supported   => [">= 1.1.0"],
+        :deprecated  => ["~>1.0.0"],
+        :url         => "https://rubygems.org/gems/rack",
+        :feed        => "https://rubygems.org/gems/rack/versions.atom"
+      },
+      :grape =>
+      {
+        :type        => :web,
+        :supported   => [">= 0.2.0"],
+        :url         => "https://rubygems.org/gems/grape",
+        :feed        => "https://rubygems.org/gems/grape/versions.atom"
+      },
+
+      # Database
+      :activerecord =>
+      {
+        :type        => :database,
+        :supported   => ["~>2.1.0", "~>2.2.0", "~>2.3.0", "~>3.0.0", "~>3.1.0", "~>3.2.0", "~>4.0.0", "~>4.1.0", "~>4.2.0"],
+        :deprecated  => ["~>2.0.0"],
+        :url         => "https://rubygems.org/gems/activerecord",
+        :feed        => "https://rubygems.org/gems/activerecord/versions.atom",
+        :notes       => ["Last supported agent for 2.0.x was 3.6.8.168"]
+      },
+      :datamapper =>
+      {
+        :type        => :database,
+        :supported   => ["~>1.0"],
+        :url         => "https://rubygems.org/gems/datamapper",
+        :feed        => "https://rubygems.org/gems/datamapper/versions.atom"
+      },
+      :sequel =>
+      {
+        :type        => :database,
+        :supported   => ["~>3.37", "~>4.0"],
+        :url         => "https://rubygems.org/gems/sequel",
+        :feed        => "https://rubygems.org/gems/sequel/versions.atom"
+      },
+      :mongo =>
+      {
+        :type        => :database,
+        :supported   => [">= 1.8.0", "< 2.0.0"],
+        :url         => "https://rubygems.org/gems/mongo",
+        :feed        => "https://rubygems.org/gems/mongo/versions.atom"
+      },
+
+      # Background Jobs
+      :resque =>
+      {
+        :type        => :background,
+        :supported   => ["~>1.23.0"],
+        :deprecated  => ["~>1.22.0"],
+        :experimental=> [">= 2.0"],
+        :url         => "https://rubygems.org/gems/resque",
+        :feed        => "https://rubygems.org/gems/resque/versions.atom"
+      },
+      :sidekiq =>
+      {
+        :type        => :background,
+        :supported   => ["~>2.8"],
+        :url         => "https://rubygems.org/gems/sidekiq",
+        :feed        => "https://rubygems.org/gems/sidekiq/versions.atom"
+      },
+      :delayed_job =>
+      {
+        :type        => :background,
+        :supported   => ["~>2.0", "~>3.0", "~>4.0"],
+        :url         => "https://rubygems.org/gems/delayed_job",
+        :feed        => "https://rubygems.org/gems/delayed_job/versions.atom"
+      },
+
+      # HTTP Clients
+      :curb =>
+      {
+        :type        => :http,
+        :supported   => [ ">= 0.8.1" ],
+        :url         => "https://rubygems.org/gems/curb",
+        :feed        => "https://rubygems.org/gems/curb/versions.atom"
+      },
+      :excon =>
+      {
+        :type        => :http,
+        :supported   => [ ">= 0.10.1" ],
+        :url         => "https://rubygems.org/gems/excon",
+        :feed        => "https://rubygems.org/gems/excon/versions.atom"
+      },
+      :httpclient =>
+      {
+        :type        => :http,
+        :supported   => [ ">= 2.2.0"],
+        :url         => "https://rubygems.org/gems/httpclient",
+        :feed        => "https://rubygems.org/gems/httpclient/versions.atom"
+      },
+      :typhoeus =>
+      {
+        :type        => :http,
+        :supported   => [ ">= 0.5.3"],
+        :url         => "https://rubygems.org/gems/typhoeus",
+        :feed        => "https://rubygems.org/gems/typhoeus/versions.atom"
+      },
+      :net_http =>
+      {
+        :type        => :http,
+        :name        => "Net::HTTP",
+        :notes       => [
+          "Supported for all agent-supported versions of Ruby.",
+          "For more information on supported HTTP clients see http://docs.mmtrix.com/docs/ruby/ruby-http-clients."]
+      },
+
+      # Other
+      :sunspot =>
+      {
+        :type        => :other,
+        :url         => "https://rubygems.org/gems/sunspot",
+        :feed        => "https://rubygems.org/gems/sunspot/versions.atom"
+      },
+      :acts_as_solr =>
+      {
+        :type        => :other,
+        :url         => "https://rubygems.org/gems/acts_as_solr",
+        :feed        => "https://rubygems.org/gems/acts_as_solr/versions.atom"
+      },
+      :dalli =>
+      {
+        :type        => :other,
+        :url         => "https://rubygems.org/gems/dalli",
+        :feed        => "https://rubygems.org/gems/dalli/versions.atom"
+      },
+      :'memcache-client' =>
+      {
+        :type        => :other,
+        :url         => "https://rubygems.org/gems/memcache-client",
+        :feed        => "https://rubygems.org/gems/memcache-client/versions.atom"
+      },
+      :authlogic =>
+      {
+        :type        => :other,
+        :url         => "https://rubygems.org/gems/authlogic",
+        :feed        => "https://rubygems.org/gems/authlogic/versions.atom"
+      },
+      :activemerchant =>
+      {
+        :type        => :other,
+        :supported   => [ ">= 1.25.0"],
+        :url         => "https://rubygems.org/gems/activemerchant",
+        :feed        => "https://rubygems.org/gems/activemerchant/versions.atom"
+      },
+    }
+  end
+end

data/lib/mmtrix/agent/synthetics_event_buffer.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/event_buffer'
+require 'mmtrix/agent/sized_buffer'
+
+module Mmtrix
+  module Agent
+    class SyntheticsEventBuffer < SizedBuffer
+
+      def append_with_reject(x)
+        @seen += 1
+        if full?
+          timestamp = timestamp_for(x)
+          latest_event = @items.max_by do |item|
+            timestamp_for(item)
+          end
+
+          if timestamp < timestamp_for(latest_event)
+            # Make room!
+            @items.delete(latest_event)
+            return [append_event(x), latest_event]
+          else
+            return [nil, x]
+          end
+        else
+          return [append_event(x), nil]
+        end
+      end
+
+      TIMESTAMP = "timestamp".freeze
+
+      def timestamp_for(event)
+        main_event, _ = event
+        main_event[TIMESTAMP]
+      end
+
+    end
+  end
+end
+

data/lib/mmtrix/agent/synthetics_monitor.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+require 'mmtrix/agent/inbound_request_monitor'
+
+module Mmtrix
+  module Agent
+    class SyntheticsMonitor < InboundRequestMonitor
+      SYNTHETICS_HEADER_KEY  = 'HTTP_X_MMTRIX_SYNTHETICS'.freeze
+
+      SUPPORTED_VERSION = 1
+      EXPECTED_PAYLOAD_LENGTH = 5
+
+      def on_finished_configuring(events)
+        events.subscribe(:before_call, &method(:on_before_call))
+      end
+
+      def on_before_call(request) #THREAD_LOCAL_ACCESS
+        encoded_header = request[SYNTHETICS_HEADER_KEY]
+        return unless encoded_header
+
+        incoming_payload = deserialize_header(encoded_header, SYNTHETICS_HEADER_KEY)
+
+        return unless incoming_payload &&
+            is_valid_payload?(incoming_payload) &&
+            is_supported_version?(incoming_payload) &&
+            is_trusted?(incoming_payload)
+
+        state = Mmtrix::Agent::TransactionState.tl_get
+        txn = state.current_transaction
+        txn.raw_synthetics_header = encoded_header
+        txn.synthetics_payload    = incoming_payload
+      end
+
+      def is_supported_version?(incoming_payload)
+        incoming_payload.first == SUPPORTED_VERSION
+      end
+
+      def is_trusted?(incoming_payload)
+        account_id = incoming_payload[1]
+        Mmtrix::Agent.config[:trusted_account_ids].include?(account_id)
+      end
+
+      def is_valid_payload?(incoming_payload)
+        incoming_payload.length == EXPECTED_PAYLOAD_LENGTH
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/system_info.rb

@@ -0,0 +1,251 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+# This module is intended to provide access to information about the host OS and
+# [virtual] machine. It intentionally does no caching and maintains no state -
+# caching should be handled by clients if needed. Methods should return nil if
+# the requested information is unavailable.
+
+require 'rbconfig'
+
+module Mmtrix
+  module Agent
+    module SystemInfo
+      def self.ruby_os_identifier
+        RbConfig::CONFIG['target_os']
+      end
+
+      def self.darwin?
+        !!(ruby_os_identifier =~ /darwin/i)
+      end
+
+      def self.linux?
+        !!(ruby_os_identifier =~ /linux/i)
+      end
+
+      def self.bsd?
+        !!(ruby_os_identifier =~ /bsd/i)
+      end
+
+      @processor_info = nil
+
+      def self.clear_processor_info
+        @processor_info = nil
+      end
+
+      def self.get_processor_info
+        if @processor_info.nil?
+          if darwin?
+            @processor_info = {
+              :num_physical_packages  => sysctl_value('hw.packages').to_i,
+              :num_physical_cores     => sysctl_value('hw.physicalcpu_max').to_i,
+              :num_logical_processors => sysctl_value('hw.logicalcpu_max').to_i
+            }
+            # in case those don't work, try backup values
+            if @processor_info[:num_physical_cores] <= 0
+              @processor_info[:num_physical_cores] = sysctl_value('hw.physicalcpu').to_i
+            end
+            if @processor_info[:num_logical_processors] <= 0
+              @processor_info[:num_logical_processors] = sysctl_value('hw.logicalcpu').to_i
+            end
+            if @processor_info[:num_logical_processors] <= 0
+              @processor_info[:num_logical_processors] = sysctl_value('hw.ncpu').to_i
+            end
+
+          elsif linux?
+            cpuinfo = proc_try_read('/proc/cpuinfo')
+            @processor_info = cpuinfo ? parse_cpuinfo(cpuinfo) : {}
+
+          elsif bsd?
+            @processor_info = {
+              :num_physical_packages  => nil,
+              :num_physical_cores     => nil,
+              :num_logical_processors => sysctl_value('hw.ncpu').to_i
+            }
+          end
+
+          # give nils for obviously wrong values
+          @processor_info.keys.each do |key|
+            value = @processor_info[key]
+            if value.is_a?(Numeric) && value <= 0
+              @processor_info[key] = nil
+            end
+          end
+        end
+
+        @processor_info
+      rescue
+        {}
+      end
+
+      def self.sysctl_value(name)
+        # make sure to redirect stderr so we don't spew if the name is unknown
+        `sysctl -n #{name} 2>/dev/null`
+      end
+
+      def self.parse_cpuinfo(cpuinfo)
+        # Build a hash of the form
+        #   { [phys_id, core_id] => num_logical_processors_on_this_core }
+        cores = Hash.new(0)
+        phys_id = core_id = nil
+
+        total_processors = 0
+
+        cpuinfo.split("\n").map(&:strip).each do |line|
+          case line
+          when /^processor\s*:/
+            cores[[phys_id, core_id]] += 1 if phys_id && core_id
+            phys_id = core_id = nil # reset these values
+            total_processors += 1
+          when /^physical id\s*:(.*)/
+            phys_id = $1.strip.to_i
+          when /^core id\s*:(.*)/
+            core_id = $1.strip.to_i
+          end
+        end
+        cores[[phys_id, core_id]] += 1 if phys_id && core_id
+
+        num_physical_packages  = cores.keys.map(&:first).uniq.size
+        num_physical_cores     = cores.size
+        num_logical_processors = cores.values.reduce(0,:+)
+
+        if num_physical_cores == 0
+          num_logical_processors = total_processors
+
+          if total_processors == 1
+            # Some older, single-core processors might not list ids,
+            # so we'll just mark them all 1.
+            num_physical_packages = 1
+            num_physical_cores    = 1
+          else
+            # We have no way of knowing how many packages or cores
+            # we have, even though we know how many processors there are.
+            num_physical_packages = nil
+            num_physical_cores    = nil
+          end
+        end
+
+        {
+          :num_physical_packages  => num_physical_packages,
+          :num_physical_cores     => num_physical_cores,
+          :num_logical_processors => num_logical_processors
+        }
+      end
+
+      def self.num_physical_packages ; get_processor_info[:num_physical_packages ] end
+      def self.num_physical_cores    ; get_processor_info[:num_physical_cores    ] end
+      def self.num_logical_processors; get_processor_info[:num_logical_processors] end
+
+      def self.processor_arch
+        RbConfig::CONFIG['target_cpu']
+      end
+
+      def self.os_version
+        proc_try_read('/proc/version')
+      end
+
+      def self.docker_container_id
+        return unless ruby_os_identifier =~ /linux/
+
+        cgroup_info = proc_try_read('/proc/self/cgroup')
+        return unless cgroup_info
+
+        parse_docker_container_id(cgroup_info)
+      end
+
+      def self.parse_docker_container_id(cgroup_info)
+        cpu_cgroup = parse_cgroup_ids(cgroup_info)['cpu']
+        return unless cpu_cgroup
+
+        container_id = case cpu_cgroup
+        # docker native driver w/out systemd (fs)
+        when %r{^/docker/([0-9a-f]+)$}                      then $1
+        # docker native driver with systemd
+        when %r{^/system\.slice/docker-([0-9a-f]+)\.scope$} then $1
+        # docker lxc driver
+        when %r{^/lxc/([0-9a-f]+)$}                         then $1
+        # not in any cgroup
+        when '/'                                            then nil
+        # in a cgroup, but we don't recognize its format
+        when %r{docker}                                     then
+          ::Mmtrix::Agent.logger.debug("Cgroup indicates docker but container_id unrecognized: '#{cpu_cgroup}'")
+          ::Mmtrix::Agent.increment_metric "Supportability/utilization/docker/error"
+          return
+        else
+          ::Mmtrix::Agent.logger.debug("Ignoring unrecognized cgroup ID format: '#{cpu_cgroup}'")
+          return
+        end
+
+        if container_id && container_id.size != 64
+          ::Mmtrix::Agent.logger.debug("Found docker container_id with invalid length: #{container_id}")
+          ::Mmtrix::Agent.increment_metric "Supportability/utilization/docker/error"
+          nil
+        else
+          container_id
+        end
+      end
+
+      def self.parse_cgroup_ids(cgroup_info)
+        cgroup_ids = {}
+
+        cgroup_info.split("\n").each do |line|
+          parts = line.split(':')
+          next unless parts.size == 3
+          _, subsystems, cgroup_id = parts
+          subsystems = subsystems.split(',')
+          subsystems.each do |subsystem|
+            cgroup_ids[subsystem] = cgroup_id
+          end
+        end
+
+        cgroup_ids
+      end
+
+      # A File.read against /(proc|sysfs)/* can hang with some older Linuxes.
+      # See https://bugzilla.redhat.com/show_bug.cgi?id=604887, RUBY-736, and
+      # https://github.com/opscode/ohai/commit/518d56a6cb7d021b47ed3d691ecf7fba7f74a6a7
+      # for details on why we do it this way.
+      def self.proc_try_read(path)
+        return nil unless File.exist?(path)
+        content = ''
+        File.open(path) do |f|
+          loop do
+            begin
+              content << f.read_nonblock(4096)
+            rescue EOFError
+              break
+            rescue Errno::EWOULDBLOCK, Errno::EAGAIN
+              content = nil
+              break # don't select file handle, just give up
+            end
+          end
+        end
+        content
+      end
+
+      def self.ram_in_mib
+        if darwin?
+          sysctl_value('hw.memsize').to_i / (1024 ** 2)
+        elsif linux?
+          meminfo = proc_try_read('/proc/meminfo')
+          parse_linux_meminfo_in_mib(meminfo)
+        elsif bsd?
+          sysctl_value('hw.realmem').to_i / (1024 ** 2)
+        else
+          ::Mmtrix::Agent.logger.debug("Unable to determine ram_in_mib for host os: #{ruby_os_identifier}")
+          nil
+        end
+      end
+
+      def self.parse_linux_meminfo_in_mib(meminfo)
+        if mem_total = meminfo[/MemTotal:\s*(\d*)\skB/,1]
+          mem_total.to_i / 1024
+        else
+          ::Mmtrix::Agent.logger.debug("Failed to parse MemTotal from /proc/meminfo: #{meminfo}")
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/mmtrix/agent/threading/agent_thread.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+# This file is distributed under Mmtrix's license terms.
+# See https://github.com/mmtrix/rpm/blob/master/LICENSE for complete details.
+
+module Mmtrix
+  module Agent
+    module Threading
+      class AgentThread
+
+        def self.create(label, &blk)
+          ::Mmtrix::Agent.logger.debug("Creating Mmtrix thread: #{label}")
+          wrapped_blk = Proc.new do
+            begin
+              blk.call
+            rescue => e
+              ::Mmtrix::Agent.logger.error("Thread #{label} exited with error", e)
+            rescue Exception => e
+              ::Mmtrix::Agent.logger.error("Thread #{label} exited with exception. Re-raising in case of interupt.", e)
+              raise
+            ensure
+              ::Mmtrix::Agent.logger.debug("Exiting Mmtrix thread: #{label}")
+            end
+          end
+
+          thread = backing_thread_class.new(&wrapped_blk)
+          thread[:mmtrix_label] = label
+          thread
+        end
+
+        # Simplifies testing if we don't directly use ::Thread.list, so keep
+        # the accessor for it here on AgentThread to use and stub.
+        def self.list
+          backing_thread_class.list
+        end
+
+        def self.bucket_thread(thread, profile_agent_code) #THREAD_LOCAL_ACCESS
+          if thread.key?(:mmtrix_label)
+            profile_agent_code ? :agent : :ignore
+          else
+            state = TransactionState.tl_state_for(thread)
+            if state.in_background_transaction?
+              :background
+            elsif state.in_web_transaction?
+              :request
+            else
+              :other
+            end
+          end
+        end
+
+        def self.scrub_backtrace(thread, profile_agent_code)
+          begin
+            bt = thread.backtrace
+          rescue Exception => e
+            ::Mmtrix::Agent.logger.debug("Failed to backtrace #{thread.inspect}: #{e.class.name}: #{e.to_s}")
+          end
+          return nil unless bt
+          bt.reject! { |t| t.include?('/mmtrix_rpm-') } unless profile_agent_code
+          bt
+        end
+
+        # To allow tests to swap out Thread for a synchronous alternative,
+        # surface the backing class we'll use from the class level.
+        @backing_thread_class = ::Thread
+
+        def self.backing_thread_class
+          @backing_thread_class
+        end
+
+        def self.backing_thread_class=(clazz)
+          @backing_thread_class = clazz
+        end
+      end
+    end
+  end
+end