tingyun_rpm 1.0.6

data/lib/ting_yun/agent/collector/middle_ware_collector/cpu_sampler.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+require 'ting_yun/agent'
+require 'ting_yun/agent/collector/middle_ware_collector/sampler'
+require 'ting_yun/support/system_info'
+
+module TingYun
+  module Agent
+    module Collector
+      class CpuSampler < ::TingYun::Agent::Collector::Sampler
+        attr_reader :last_time
+
+        named :cpu
+
+        def initialize
+          @last_time = nil
+          @processor_count = TingYun::Support::SystemInfo.num_logical_processors
+          if @processor_count.nil?
+            TingYun::Agent.logger.warn("Failed to determine processor count, assuming 1")
+            @processor_count = 1
+          end
+          poll
+        end
+
+        def record_user_util(value)
+          TingYun::Agent.record_metric("CPU/NULL/UserUtilization", value)
+        end
+
+        def record_system_util(value)
+          TingYun::Agent.record_metric("CPU/System/Utilization", value)
+        end
+
+        def record_usertime(value)
+          TingYun::Agent.record_metric("CPU/NULL/UserTime", value)
+        end
+
+        def record_systemtime(value)
+          TingYun::Agent.record_metric("CPU/System Time", value)
+        end
+
+        def poll
+          now = Time.now
+          t = Process.times
+          if @last_time
+            elapsed = now - @last_time
+            return if elapsed < 1 # Causing some kind of math underflow
+
+            usertime = t.utime - @last_utime
+            # systemtime = t.stime - @last_stime
+
+            # record_systemtime(systemtime) if systemtime >= 0
+            record_usertime(usertime) if usertime >= 0
+
+            # Calculate the true utilization by taking cpu times and dividing by
+            # elapsed time X processor_count.
+
+            record_user_util(usertime / (elapsed * @processor_count))
+            # record_system_util(systemtime / (elapsed * @processor_count))
+          end
+          @last_utime = t.utime
+          @last_stime = t.stime
+          @last_time = now
+        end
+      end
+    end
+  end
+end
+

data/lib/ting_yun/agent/collector/middle_ware_collector/memory_sampler.rb

@@ -0,0 +1,139 @@
+# encoding: utf-8
+
+require 'ting_yun/agent'
+require 'ting_yun/agent/collector/middle_ware_collector/sampler'
+require 'ting_yun/support/system_info'
+
+module TingYun
+  module Agent
+    module Collector
+      class MemorySampler < ::TingYun::Agent::Collector::Sampler
+        named :memory
+
+        attr_accessor :sampler
+
+        def initialize
+          # macos, linux, solaris
+          if platform =~ /linux/
+            @sampler = ProcStatus.new
+            if !@sampler.can_run?
+              ::TingYun::Agent.logger.debug "Error attempting to use /proc/#{$$}/status file for reading memory. Using ps command instead."
+              @sampler = ShellPS.new("ps -o rsz")
+            else
+              ::TingYun::Agent.logger.debug "Using /proc/#{$$}/status for reading process memory."
+            end
+          elsif platform =~ /darwin9/ # 10.5
+            @sampler = ShellPS.new("ps -o rsz")
+          elsif platform =~ /darwin1\d+/ # >= 10.6
+            @sampler = ShellPS.new("ps -o rss")
+          elsif platform =~ /freebsd/
+            @sampler = ShellPS.new("ps -o rss")
+          elsif platform =~ /solaris/
+            @sampler = ShellPS.new("/usr/bin/ps -o rss -p")
+          end
+
+          raise Unsupported, "Unsupported platform for getting memory: #{platform}" if @sampler.nil?
+          raise Unsupported, "Unable to run #{@sampler}" unless @sampler.can_run?
+        end
+
+        def self.supported_on_this_platform?
+          platform =~ /linux|darwin|freebsd|solaris/
+        end
+
+        def platform
+          TingYun::Support::SystemInfo.ruby_os_identifier.downcase
+        end
+
+        def self.platform
+          TingYun::Support::SystemInfo.ruby_os_identifier.downcase
+        end
+
+        def poll
+          sample = @sampler.get_sample
+          if sample
+            TingYun::Agent.record_metric("Memory/NULL/PhysicalUsed", sample)
+          end
+        end
+
+        class Base
+          def initialize
+            @broken = false
+          end
+
+          def can_run?
+            return false if @broken
+            m = get_memory rescue nil
+            m && m > 0
+          end
+
+          def get_memory
+            raise 'Implement in the subclass'
+          end
+
+          def get_sample
+            return nil if @broken
+            begin
+              m = get_memory
+              if m.nil?
+                ::TingYun::Agent.logger.warn "Unable to get the resident memory for process #{$$}.  Disabling memory sampler."
+                @broken = true
+              end
+              return m
+            rescue => e
+              ::TingYun::Agent.logger.warn "Unable to get the resident memory for process #{$$}. Disabling memory sampler.", e
+              @broken = true
+              return nil
+            end
+          end
+        end
+
+        class ShellPS < Base
+          def initialize(command)
+            super()
+            @command = command
+          end
+
+          # Returns the amount of resident memory this process is using in MB
+          #
+          def get_memory
+            process = $$
+            memory = `#{@command} #{process}`.split("\n")[1].to_f / 1024.0 rescue nil
+            # if for some reason the ps command doesn't work on the resident os,
+            # then don't execute it any more.
+            raise "Faulty command: `#{@command} #{process}`" if memory.nil? || memory <= 0
+            memory
+          end
+
+          def to_s
+            "shell command sampler: #{@command}"
+          end
+        end
+
+        # ProcStatus
+        #
+        # A class that samples memory by reading the file /proc/$$/status, which is specific to linux
+        #
+        class ProcStatus < Base
+          # Returns the amount of resident memory this process is using in MB
+          def get_memory
+            proc_status = File.open(proc_status_file, "r") {|f| f.read_nonblock(4096).strip }
+            if proc_status =~ /RSS:\s*(\d+) kB/i
+              return $1.to_f / 1024.0
+            end
+            raise "Unable to find RSS in #{proc_status_file}"
+          end
+
+          def proc_status_file
+            "/proc/#{$$}/status"
+          end
+
+          def to_s
+            "proc status file sampler: #{proc_status_file}"
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/ting_yun/agent/collector/middle_ware_collector/middle_ware.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+module TingYun
+  module Agent
+    module Collector
+      module MiddleWare
+        require 'ting_yun/agent/collector/middle_ware_collector/cpu_sampler'
+        require 'ting_yun/agent/collector/middle_ware_collector/memory_sampler'
+      end
+    end
+  end
+end
+

data/lib/ting_yun/agent/collector/middle_ware_collector/sampler.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+
+require 'ting_yun/agent'
+
+module TingYun
+  module Agent
+    module Collector
+      class Sampler
+
+        class Unsupported < StandardError;  end
+
+        attr_reader :id
+        @sampler_classes = []
+
+        def self.named(new_name)
+          @name = new_name
+        end
+
+        def self.name
+          @name
+        end
+
+        def self.inherited(subclass)
+          @sampler_classes << subclass
+        end
+
+        # Override with check.  Called before instantiating.
+        def self.supported_on_this_platform?
+          true
+        end
+
+        def self.enabled?
+          if @name
+            config_key = "disable_#{@name}_sampler"
+            !(::TingYun::Agent.config[config_key])
+          else
+            true
+          end
+        end
+
+        def self.sampler_classes
+          @sampler_classes
+        end
+
+        # The ID passed in here is unused by our code, but is preserved in case
+        # we have clients who are defining their own subclasses of this class, and
+        # expecting to be able to call super with an ID.
+        def initialize(id=nil)
+          @id = id || self.class.name
+        end
+
+        def poll
+          raise "Implement in the subclass"
+        end
+      end
+    end
+  end
+end
+

data/lib/ting_yun/agent/collector/middle_ware_collector.rb

@@ -0,0 +1,80 @@
+# encoding: utf-8
+
+require 'ting_yun/agent'
+require 'ting_yun/agent/transaction/transaction_state'
+require 'ting_yun/agent/collector/middle_ware_collector/sampler'
+
+
+module TingYun
+  module Agent
+    module Collector
+      class MiddleWareCollector
+
+        include Enumerable
+
+        def initialize(event_listener)
+          @samplers = []
+          @event_listener = event_listener
+          @event_listener.subscribe(:middleware_harvest) { samplers_poll }
+        end
+
+        def samplers_poll
+          @samplers.delete_if do |sampler|
+            begin
+              sampler.poll
+              false # it's okay.  don't delete it.
+            rescue => e
+              ::TingYun::Agent.logger.warn("Removing #{sampler} from list", e)
+              true # remove the sampler
+            end
+          end
+        end
+
+        def each(&blk)
+          @samplers.each(&blk)
+        end
+
+        def clear()
+          @samplers.clear
+        end
+
+        def sampler_class_registered?(sampler_class)
+          self.any? { |s| s.class == sampler_class }
+        end
+
+        def register_sampler(sampler_class)
+          supported = sampler_class.supported_on_this_platform?
+          enabled = sampler_class.enabled?
+          if supported && enabled
+            if !sampler_class_registered?(sampler_class)
+              sampler = sampler_class.new
+
+              @samplers << sampler
+              ::TingYun::Agent.logger.debug("Registered #{sampler_class.name} for harvest time sampling.")
+            else
+              ::TingYun::Agent.logger.warn("Ignoring addition of #{sampler_class.name} because it is already registered.")
+            end
+          else
+            ::TingYun::Agent.logger.debug("#{sampler_class.name} not supported on this platform.")
+          end
+        rescue TingYun::Agent::Collector::Sampler::Unsupported => e
+          ::TingYun::Agent.logger.info("#{sampler_class.name} not available: #{e}")
+        rescue => e
+          ::TingYun::Agent.logger.error("Error registering sampler:", e)
+        end
+
+
+        # adds samplers to the sampler collection so that they run every
+        # minute. This is dynamically recognized by any class that
+        def load_samplers
+          TingYun::Agent::Collector::Sampler.sampler_classes.each do |subclass|
+            register_sampler(subclass)
+          end
+        end
+      end
+    end
+  end
+end
+
+
+

data/lib/ting_yun/agent/collector/sql_sampler.rb

@@ -0,0 +1,299 @@
+# encoding: utf-8
+
+require 'ting_yun/agent/database'
+require 'ting_yun/agent/transaction/transaction_state'
+require 'ting_yun/support/coerce'
+require 'ting_yun/metrics/stats'
+require 'ting_yun/support/helper'
+module TingYun
+  module Agent
+    module Collector
+      # This class contains the logic of recording slow SQL traces, which may
+      # represent multiple aggregated SQL queries.
+      #
+      # A slow SQL trace consists of a collection of SQL instrumented SQL queries
+      # that all normalize to the same text. For example, the following two
+      # queries would be aggregated together into a single slow SQL trace:
+      #
+      #   SELECT * FROM table WHERE id=42
+      #   SELECT * FROM table WHERE id=1234
+      #
+      # Each slow SQL trace keeps track of the number of times the same normalized
+      # query was seen, the min, max, and total time spent executing those
+      # queries, and an example backtrace from one of the aggregated queries.
+      #
+      class SqlSampler
+
+        MAX_SAMPLES = 10
+
+        attr_reader :sql_traces
+
+        def initialize
+          @sql_traces = {}
+          @samples_lock = Mutex.new
+        end
+
+        def enabled?
+          Agent.config[:'nbs.action_tracer.enabled'] &&
+              Agent.config[:'nbs.action_tracer.slow_sql'] &&
+                  TingYun::Agent::Database.should_record_sql?('nbs.action_tracer.record_sql')
+        end
+
+        def on_start_transaction(state, uri=nil)
+          return unless enabled?
+
+          state.sql_sampler_transaction_data = TransactionSqlData.new
+
+          if Agent.config[:'nbs.action_tracer.slow_sql'] && state.sql_sampler_transaction_data
+            state.sql_sampler_transaction_data.set_transaction_info(uri)
+          end
+        end
+
+        # duration{:type => sec}
+        def notice_sql(sql, metric_name, config, duration, state=nil, explainer=nil) #THREAD_LOCAL_ACCESS sometimes
+          start_time = Time.now.to_f
+          state ||= TingYun::Agent::TransactionState.tl_get
+          data = state.sql_sampler_transaction_data
+          return unless data
+
+          if state.sql_recorded? && !metric_name.nil?
+            if duration*1000 > TingYun::Agent.config[:'nbs.action_tracer.slow_sql_threshold']
+              if duration*1000 > TingYun::Agent.config[:'nbs.action_tracer.stack_trace_threshold']
+                backtrace = (caller.reject! { |t| t.include?('tingyun_rpm') }).join("\n")
+              else
+                backtrace = ''
+              end
+              statement = TingYun::Agent::Database::Statement.new(sql, config, explainer)
+              data.sql_data << SlowSql.new(statement, metric_name, duration, start_time, backtrace)
+            end
+          end
+        end
+
+
+        def on_finishing_transaction(state, name)
+          return unless enabled?
+
+          transaction_sql_data = state.sql_sampler_transaction_data
+          return unless transaction_sql_data
+
+          transaction_sql_data.set_transaction_name(name)
+
+          save_slow_sql(transaction_sql_data)
+        end
+
+        def save_slow_sql(data)
+          if data.sql_data.size > 0
+            @samples_lock.synchronize do
+              ::TingYun::Agent.logger.debug "Examining #{data.sql_data.size} slow transaction sql statement(s)"
+              save data
+            end
+          end
+        end
+
+        def save (transaction_sql_data)
+          action_metric_name = transaction_sql_data.action_metric_name
+          uri                = transaction_sql_data.uri
+
+          transaction_sql_data.sql_data.each do |sql_item|
+            normalized_sql = sql_item.normalize
+            sql_trace = @sql_traces[normalized_sql]
+            if sql_trace
+              sql_trace.aggregate(sql_item, action_metric_name, uri)
+            else
+              if has_room?
+                sql_trace = SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
+              elsif should_add_trace?(sql_item)
+                remove_shortest_trace
+                sql_trace = SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
+              end
+
+              if sql_trace
+                @sql_traces[normalized_sql] = sql_trace
+              end
+
+            end
+          end
+        end
+
+        # this should always be called under the @samples_lock
+        def has_room?
+          @sql_traces.size < MAX_SAMPLES
+        end
+
+        # this should always be called under the @samples_lock
+        def should_add_trace?(sql_item)
+          @sql_traces.any? do |(_, existing_trace)|
+            existing_trace.max_call_time < sql_item.duration
+          end
+        end
+
+        # this should always be called under the @samples_lock
+        def remove_shortest_trace
+          shortest_key, _ = @sql_traces.min_by { |(_, trace)| trace.max_call_time }
+          @sql_traces.delete(shortest_key)
+        end
+
+
+        def harvest!
+          return [] unless enabled?
+
+          slowest = []
+          @samples_lock.synchronize do
+            slowest = @sql_traces.values
+            @sql_traces = {}
+          end
+          slowest.each {|trace| trace.prepare_to_send }
+          slowest
+        end
+
+        def reset!
+          @samples_lock.synchronize do
+            @sql_traces = {}
+          end
+        end
+
+        def merge!
+          @samples_lock.synchronize do
+            sql_traces.each do |trace|
+              existing_trace = @sql_traces[trace.sql]
+              if existing_trace
+                existing_trace.aggregate_trace(trace)
+              else
+                @sql_traces[trace.sql] = trace
+              end
+            end
+          end
+        end
+
+
+      end
+
+      class TransactionSqlData
+        attr_reader :action_metric_name
+        attr_reader :uri
+        attr_reader :sql_data
+
+        def initialize
+          @sql_data = []
+        end
+
+        def set_transaction_info(uri)
+          @uri = uri
+        end
+
+        def set_transaction_name(name)
+          @action_metric_name = name
+        end
+      end
+
+      class SlowSql
+        attr_reader :statement
+        attr_reader :metric_name
+        attr_reader :duration
+        attr_reader :backtrace
+        attr_reader :start_time
+
+
+        def initialize(statement, metric_name, duration, t0,  backtrace=nil)
+          @start_time = t0
+          @statement = statement
+          @metric_name = metric_name
+          @duration = duration
+          @backtrace = backtrace
+        end
+
+        def sql
+          statement.sql
+        end
+
+        def obfuscate
+          TingYun::Agent::Database.obfuscate_sql(statement)
+        end
+
+
+        def normalize
+          TingYun::Agent::Database::Obfuscator.instance.default_sql_obfuscator(statement)
+        end
+
+        def explain
+          if statement.config && statement.explainer
+            TingYun::Agent::Database.explain_sql(statement.sql, statement.config, statement.explainer)
+          end
+        end
+
+        # We can't serialize the explainer, so clear it before we transmit
+        def prepare_to_send
+          statement.explainer = nil
+        end
+      end
+
+
+
+      class SqlTrace < TingYun::Metrics::Stats
+
+        attr_reader :action_metric_name
+        attr_reader :uri
+        attr_reader :sql
+        attr_reader :slow_sql
+        attr_reader :params
+
+        def initialize(normalized_query, slow_sql, action_name, uri)
+          super()
+          @params = {}
+
+          @action_metric_name = action_name
+          @slow_sql = slow_sql
+          @sql = normalized_query
+          @uri = uri
+          @params[:stacktrace] = slow_sql.backtrace
+          record_data_point(float(slow_sql.duration))
+        end
+
+        def aggregate(slow_sql, action_name, uri)
+          if slow_sql.duration > max_call_time
+            @action_metric_name = action_name
+            @slow_sql = slow_sql
+            @uri = uri
+            @params[:stacktrace] = slow_sql.backtrace
+          end
+          record_data_point(float(slow_sql.duration))
+        end
+
+        def aggregate_trace(trace)
+          aggregate(trace.slow_sql, trace.path, trace.url)
+        end
+
+        def prepare_to_send
+          @sql = @slow_sql.sql unless need_to_obfuscate?
+          @params[:explainPlan] = @slow_sql.explain if need_to_explain?
+        end
+
+        def need_to_obfuscate?
+          Agent.config[:'nbs.action_tracer.record_sql'].to_s == 'obfuscated'
+        end
+
+        def need_to_explain?
+          Agent.config[:'nbs.action_tracer.explain_enabled'] &&  @slow_sql.duration * 1000 > TingYun::Agent.config[:'nbs.action_tracer.explain_threshold']
+        end
+
+
+        include TingYun::Support::Coerce
+
+        def to_collector_array(encoder)
+          [
+              @slow_sql.start_time,
+              string(@action_metric_name),
+              string(@slow_sql.metric_name),
+              string(@uri),
+              string(@sql),
+              int(@call_count),
+              TingYun::Helper.time_to_millis(@total_call_time),
+              TingYun::Helper.time_to_millis(@max_call_time),
+              TingYun::Helper.time_to_millis(@min_call_time),
+              encoder.encode(@params)
+          ]
+        end
+      end
+    end
+  end
+end