tingyun_rpm 1.1.4.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3752c62ac9fc5fded9a78b273cf8d82254d07499
-  data.tar.gz: 92668b61dc19882c8063fcbfe1f38a7019f4f574
+  metadata.gz: 1e1698a0cac85df291567637e7d8905c7dd5d30b
+  data.tar.gz: 504578a77dac8308f1f43d6cec9933da44778a57
 SHA512:
-  metadata.gz: 3612d7ea6911dd1d38fb3e868f6fb9d753d3d0cd494c7fb770543c0263fa150fca4e09e2cb165f6ebc159274f92a25e8f27bc4dd46ebabc0f511e00da060b4a5
-  data.tar.gz: f9bb466fabbb9be45e1353d751299a45476a64beb32343c5d5351ae5cda9a7e3c74c5d2efee753f8037e466527dddd505239fcf6a096727a1bcc09dc3a0c563c
+  metadata.gz: e067d0a87a03f9428e811cea75288256ad09cc992ead66d89337ca6966d7428acaa6c2a781a67bf91b77ef62a20057c1d65445dbe6640bc686f43b0ceb52c0ff
+  data.tar.gz: 9246032c6e6841630838a20713a86355a4f73b5085ff3ce1f066988646409dc4b688f2fde31b3fa4d2e4f0e72ea8f6cbb54bf8f070d42ab8f1feae63c33cece7

data/.gitignore

@@ -11,4 +11,4 @@ log/
 .ruby-version
 .idea/
 todo
-
+.DS_Store

data/Guardfile

@@ -23,3 +23,13 @@ guard :minitest, :test_folders => ['test/ting_yun'], :all_after_pass => false do
   watch('test/agent_helper.rb') { "test/tingyun" }
 
 end
+
+guard "rubycritic" do
+  watch(%r{^app/(.+)\.rb$})
+  watch(%r{^lib/(.+)\.rb$})
+end
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+

data/lib/ting_yun/agent.rb

@@ -46,6 +46,7 @@ module TingYun
     end
 
 
+
     # Record a value for the given metric name.
     #
     # This method should be used to record event-based metrics such as method

data/lib/ting_yun/agent/agent.rb

@@ -9,6 +9,7 @@ require 'ting_yun/agent/event/event_listener'
 require 'ting_yun/agent/dispatcher'
 require 'ting_yun/agent/collector/middle_ware_collector'
 require 'ting_yun/agent/cross_app/cross_app_monitor'
+require 'ting_yun/agent/collector/transaction_sampler'
 
 
 # The Agent is a singleton that is instantiated when the plugin is
@@ -25,7 +26,7 @@ module TingYun
       end
 
       # service for communicating with collector
-      attr_accessor :service, :cross_app_monitor
+      attr_accessor :service, :cross_app_monitor, :middleware
       attr_reader :events
 
       extend ClassMethods
@@ -37,11 +38,11 @@ module TingYun
         @environment_report = nil
         @service = TingYunService.new
         @connect_state = :pending #[:pending, :connected, :disconnected]
-        @connect_attempts = 0
         @events  = TingYun::Agent::Event::EventListener.new
         @after_fork_lock = Mutex.new
         @dispatcher = TingYun::Agent::Dispatcher.new(@events)
         @cross_app_monitor = TingYun::Agent::CrossAppMonitor.new(@events)
+        @middleware = TingYun::Agent::Collector::MiddleWareCollector.new(@events)
 
         init_containers
       end
@@ -51,7 +52,7 @@ module TingYun
         return unless agent_should_start?
         log_startup
         check_config_and_start_agent
-        log_version_and_pid
+        TingYun::Agent.logger.debug "Ting Yun Ruby Agent #{TingYun::VERSION::STRING} Initialized: pid = #{$$}" # log_version_and_pid
       end
 
       # Attempt a graceful shutdown of the agent, flushing any remaining
@@ -77,39 +78,28 @@ module TingYun
 
       def connect!(option={})
         defaults = {
-            :keep_retrying => ::TingYun::Agent.config[:keep_retrying],
-            :force_reconnect => ::TingYun::Agent.config[:force_reconnect]
+            :force_reconnect => ::TingYun::Agent.config[:force_reconnect],
+            :keep_retrying => ::TingYun::Agent.config[:keep_retrying]
         }
         opts = defaults.merge(option)
         return unless should_connect?(opts[:force_reconnect])
         TingYun::Agent.logger.debug "Connecting Process to Ting Yun: #$0"
         query_server_for_configuration
-        @connected_pid = $$
         @connect_state = :connected
-      rescue TingYun::Support::Exception::LicenseException => e
-        handle_license_error(e)
-      rescue TingYun::Support::Exception::UnrecoverableAgentException => e
-        handle_unrecoverable_agent_error(e)
-      rescue StandardError, Timeout::Error, TingYun::Support::Exception::ServerConnectionException, TingYun::Support::Exception::AgentEnableException => e
-        log_error(e)
-        if TingYun::Agent.config[:keep_retrying]
-          note_connect_failure
+      rescue Exception => error
+        ::TingYun::Agent.logger.error "Exception of unexpected type during Agent#connect! :", error
+        log_error(error)
+        if opts[:keep_retrying]
           ::TingYun::Agent.logger.info "Will re-attempt in 60 seconds"
-          sleep 60
-          retry
-        else
-          disconnect
+          raise
         end
-      rescue Exception => e
-        ::TingYun::Agent.logger.error "Exception of unexpected type during Agent#connect():", e
-        raise
       end
 
 
       def install_exit_handler
         TingYun::Agent.logger.debug("Installing at_exit handler")
         at_exit do
-          if need_exit_code_workaround?
+          if defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby" && RUBY_VERSION.match(/^1\.9/)
             exit_status = $!.status if $!.is_a?(SystemExit)
             shutdown
             exit exit_status if exit_status
@@ -119,9 +109,6 @@ module TingYun
         end
       end
 
-      def need_exit_code_workaround?
-        defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby" && RUBY_VERSION.match(/^1\.9/)
-      end
 
       def untraced_graceful_disconnect
         begin
@@ -130,10 +117,12 @@ module TingYun
               transmit_data
             end
           end
-        rescue => e
-          ::TingYun::Agent.logger.error e
+        rescue => error
+          ::TingYun::Agent.logger.error error
         end
       end
+
+
     end
   end
 end

data/lib/ting_yun/agent/collector/error_collector.rb

@@ -45,7 +45,7 @@ module TingYun
             metric_names = [ERRORS_ALL]
             return metric_names unless txn
 
-            if txn.recording_web_transaction?
+            if TingYun::Agent::Transaction.recording_web_transaction?
               metric_names << ERRORS_ALL_WEB
             else
               metric_names << ERRORS_ALL_BACK_GROUND
@@ -72,7 +72,7 @@ module TingYun
         def notice_error(exception, options={})
           tag_exception(exception)
           state = ::TingYun::Agent::TransactionState.tl_get
-          increment_error_count!(state, exception, options)
+          increment_error_count(state)
           noticed_error = create_noticed_error(exception, options)
           if noticed_error.is_external_error
             external_error_array.add_to_error_queue(noticed_error)
@@ -85,7 +85,7 @@ module TingYun
         end
 
         # Increments a statistic that tracks total error rate
-        def increment_error_count!(state, exception, options={})
+        def increment_error_count(state)
           txn = state.current_transaction
 
           metric_names = aggregated_metric_names(txn)
@@ -102,26 +102,15 @@ module TingYun
         EMPTY_STRING = ''.freeze
 
         def create_noticed_error(exception, options)
-          error_metric = options.delete(:metric_name) || EMPTY_STRING
-
+          attributes = options[:attributes]
+          error_metric = attributes.agent_attributes[:metric_name] || EMPTY_STRING
           noticed_error = TingYun::Agent::Collector::NoticedError.new(error_metric, exception)
-          noticed_error.request_uri = options.delete(:uri) || EMPTY_STRING
-          noticed_error.request_port = options.delete(:port)
-          noticed_error.attributes  = options.delete(:attributes)
-
-          noticed_error.file_name   = sense_method(exception, :file_name)
-          noticed_error.line_number = sense_method(exception, :line_number)
+          noticed_error.attributes  = attributes
           noticed_error.stack_trace = extract_stack_trace(exception)
-
-          noticed_error.attributes_from_notice_error = options.delete(:custom_params) || {}
-
-          # Any options that are passed to notice_error which aren't known keys
-          # get treated as custom attributes, so merge them into that hash.
-          noticed_error.attributes_from_notice_error.merge!(options)
-
           noticed_error
         end
 
+
         def skip_notice_error?(exception)
           exception_tagged?(exception)
         end

data/lib/ting_yun/agent/collector/error_collector/noticed_error.rb

@@ -12,26 +12,28 @@ module TingYun
     module Collector
       class NoticedError
 
+
+
         attr_accessor :metric_name, :timestamp, :message, :exception_class_name,
-                      :request_uri, :request_port, :file_name, :line_number,
                       :stack_trace, :attributes_from_notice_error, :attributes,
-                      :count_error, :thread_name, :is_external_error, :external_metric_name, :code, :trace
-
+                      :count_error, :is_external_error, :external_metric_name, :code, :trace
 
         attr_reader :exception_id, :is_internal
 
 
         def initialize(metric_name, exception, timestamp = Time.now)
+          @metric_name = metric_name
+          @timestamp = timestamp
           @stack_trace = []
           @count_error = 1
           @exception_id = exception.object_id
-          @metric_name = metric_name
-          @timestamp = timestamp
           @exception_class_name = exception.is_a?(Exception) ? exception.class.name : 'Error'
-          @external_metric_name = exception.instance_variable_get :@tingyun_klass
-          @is_external_error = exception.instance_variable_get :@tingyun_external
-          @code = exception.instance_variable_get :@tingyun_code
-          @trace = exception.instance_variable_get :@tingyun_trace
+          @is_external_error = exception.respond_to?(:tingyun_external)? exception.tingyun_external : false
+          if @is_external_error
+            @external_metric_name = exception.tingyun_klass
+            @code = exception.tingyun_code
+            @trace = exception.tingyun_trace
+          end
           # It's critical that we not hold onto the exception class constant in this
           # class. These objects get serialized for Resque to a process that might
           # not have the original exception class loaded, so do all processing now
@@ -62,10 +64,13 @@ module TingYun
 
 
         def ==(other)
-          if other.respond_to?(:exception_id)
-            exception_id == other.exception_id
+          return true if other.respond_to?(:exception_id) && exception_id == other.exception_id
+
+          if metric_name == other.metric_name && message == other.message
+            @count_error = count_error + 1
+            return true
           else
-            false
+            return false
           end
         end
 
@@ -88,23 +93,23 @@ module TingYun
              string(exception_class_name),
              string(message),
              count_error,
-             string(request_uri),
+             string(attributes.agent_attributes[:request_path]||metric_name),
              encoder.encode(error_params)
             ]
           end
         end
 
         def error_params
-         hash = {
+          hash = {
               :params => custom_params
           }
-         if is_external_error
-           hash[:stacktrace] = trace
-         else
-           hash[:stacktrace] = stack_trace
-           hash[:requestParams] = request_params
-         end
-         hash
+          if is_external_error
+            hash[:stacktrace] = trace
+          else
+            hash[:stacktrace] = stack_trace
+            hash[:requestParams] = request_params
+          end
+          hash
         end
 
         def custom_params

data/lib/ting_yun/agent/collector/middle_ware_collector/cpu_sampler.rb

@@ -46,16 +46,11 @@ module TingYun
             return if elapsed < 1 # Causing some kind of math underflow
 
             usertime = t.utime - @last_utime
-            # systemtime = t.stime - @last_stime
 
-            # record_systemtime(systemtime) if systemtime >= 0
-            record_usertime(usertime) if usertime >= 0
-
-            # Calculate the true utilization by taking cpu times and dividing by
-            # elapsed time X processor_count.
-
-            record_user_util((usertime * 100) / (elapsed * @processor_count))
-            # record_system_util(systemtime / (elapsed * @processor_count))
+            if usertime >= 0
+              record_usertime(usertime)
+              record_user_util((usertime * 100) / (elapsed * @processor_count))
+            end
           end
           @last_utime = t.utime
           @last_stime = t.stime

data/lib/ting_yun/agent/collector/sql_sampler.rb

@@ -2,9 +2,11 @@
 
 require 'ting_yun/agent/database'
 require 'ting_yun/agent/transaction/transaction_state'
-require 'ting_yun/support/coerce'
-require 'ting_yun/metrics/stats'
-require 'ting_yun/support/helper'
+
+
+require 'ting_yun/agent/collector/sql_sampler/transaction_sql_data'
+require 'ting_yun/agent/collector/sql_sampler/slow_sql'
+require 'ting_yun/agent/collector/sql_sampler/sql_trace'
 module TingYun
   module Agent
     module Collector
@@ -21,7 +23,7 @@ module TingYun
       # Each slow SQL trace keeps track of the number of times the same normalized
       # query was seen, the min, max, and total time spent executing those
       # queries, and an example backtrace from one of the aggregated queries.
-      #
+
       class SqlSampler
 
         MAX_SAMPLES = 10
@@ -33,47 +35,32 @@ module TingYun
           @samples_lock = Mutex.new
         end
 
-        def enabled?
-          Agent.config[:'nbs.action_tracer.enabled'] &&
-              Agent.config[:'nbs.action_tracer.slow_sql'] &&
-                  TingYun::Agent::Database.should_record_sql?('nbs.action_tracer.record_sql')
-        end
-
-        def on_start_transaction(state, uri=nil)
-          return unless enabled?
-
-          state.sql_sampler_transaction_data = TransactionSqlData.new
+        def self.on_start_transaction(state, uri)
+          return unless TingYun::Agent::Database.sql_sampler_enabled?
 
-          if Agent.config[:'nbs.action_tracer.slow_sql'] && state.sql_sampler_transaction_data
-            state.sql_sampler_transaction_data.set_transaction_info(uri)
-          end
+          state.init_sql_transaction(::TingYun::Agent::Collector::TransactionSqlData.new(uri))
         end
 
         # duration{:type => sec}
-        def notice_sql(sql, metric_name, config, duration, state=nil, explainer=nil) #THREAD_LOCAL_ACCESS sometimes
+        def self.notice_sql(sql, metric_name, config, duration, state=nil, explainer=nil, binds=[], name="SQL") #THREAD_LOCAL_ACCESS sometimes
           start_time = Time.now.to_f
           state ||= TingYun::Agent::TransactionState.tl_get
           data = state.sql_sampler_transaction_data
           return unless data
-
-          if state.sql_recorded? && !metric_name.nil?
-            if duration*1000 > TingYun::Agent.config[:'nbs.action_tracer.slow_sql_threshold']
-              if duration*1000 > TingYun::Agent.config[:'nbs.action_tracer.stack_trace_threshold']
-                backtrace = (caller.reject! { |t| t.include?('tingyun_rpm') })
-                backtrace = backtrace.first(40) if backtrace.length > 40
-                backtrace = backtrace.join("\n")
-              else
-                backtrace = ''
-              end
-              statement = TingYun::Agent::Database::Statement.new(sql, config, explainer)
-              data.sql_data << SlowSql.new(statement, metric_name, duration, start_time, backtrace)
+          threshold = duration*1000
+          if threshold > TingYun::Agent.config[:'nbs.action_tracer.slow_sql_threshold'] && state.sql_recorded?
+            backtrace = ''
+            if threshold > TingYun::Agent.config[:'nbs.action_tracer.stack_trace_threshold']
+              backtrace = caller.reject! { |t| t.include?('tingyun_rpm') }
+              backtrace = backtrace.first(20).join("\n")
             end
+            statement = TingYun::Agent::Database::Statement.new(sql, config, explainer, binds, name)
+            data.sql_data << ::TingYun::Agent::Collector::SlowSql.new(statement, metric_name, duration, start_time, backtrace)
           end
         end
 
-
         def on_finishing_transaction(state, name)
-          return unless enabled?
+          return unless TingYun::Agent::Database.sql_sampler_enabled?
 
           transaction_sql_data = state.sql_sampler_transaction_data
           return unless transaction_sql_data
@@ -84,16 +71,17 @@ module TingYun
         end
 
         def save_slow_sql(data)
-          if data.sql_data.size > 0
+          size = data.sql_data.size
+          if size > 0
             @samples_lock.synchronize do
-              ::TingYun::Agent.logger.debug "Examining #{data.sql_data.size} slow transaction sql statement(s)"
+              ::TingYun::Agent.logger.debug "Examining #{size} slow transaction sql statement(s)"
               save data
             end
           end
         end
 
         def save (transaction_sql_data)
-          action_metric_name = transaction_sql_data.action_metric_name
+          action_metric_name = transaction_sql_data.metric_name
           uri                = transaction_sql_data.uri
 
           transaction_sql_data.sql_data.each do |sql_item|
@@ -103,16 +91,14 @@ module TingYun
               sql_trace.aggregate(sql_item, action_metric_name, uri)
             else
               if has_room?
-                sql_trace = SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
-              elsif should_add_trace?(sql_item)
-                remove_shortest_trace
-                sql_trace = SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
-              end
-
-              if sql_trace
-                @sql_traces[normalized_sql] = sql_trace
+                @sql_traces[normalized_sql] = ::TingYun::Agent::Collector::SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
+              else
+                min, max = @sql_traces.minmax_by { |(_, trace)| trace.max_call_time }
+                if max.last.max_call_time < sql_item.duration
+                  @sql_traces.delete(min.first)
+                  @sql_traces[normalized_sql] = ::TingYun::Agent::Collector::SqlTrace.new(normalized_sql, sql_item, action_metric_name, uri)
+                end
               end
-
             end
           end
         end
@@ -122,23 +108,9 @@ module TingYun
           @sql_traces.size < MAX_SAMPLES
         end
 
-        # this should always be called under the @samples_lock
-        def should_add_trace?(sql_item)
-          @sql_traces.any? do |(_, existing_trace)|
-            existing_trace.max_call_time < sql_item.duration
-          end
-        end
-
-        # this should always be called under the @samples_lock
-        def remove_shortest_trace
-          shortest_key, _ = @sql_traces.min_by { |(_, trace)| trace.max_call_time }
-          @sql_traces.delete(shortest_key)
-        end
-
 
         def harvest!
-          return [] unless enabled?
-
+          return [] unless TingYun::Agent::Database.sql_sampler_enabled?
           slowest = []
           @samples_lock.synchronize do
             slowest = @sql_traces.values
@@ -159,143 +131,15 @@ module TingYun
             sql_traces.each do |trace|
               existing_trace = @sql_traces[trace.sql]
               if existing_trace
-                existing_trace.aggregate_trace(trace)
+                existing_trace.aggregate(trace.slow_sql, trace.path, trace.url)
               else
                 @sql_traces[trace.sql] = trace
               end
             end
           end
         end
-
-
-      end
-
-      class TransactionSqlData
-        attr_reader :action_metric_name
-        attr_reader :uri
-        attr_reader :sql_data
-
-        def initialize
-          @sql_data = []
-        end
-
-        def set_transaction_info(uri)
-          @uri = uri
-        end
-
-        def set_transaction_name(name)
-          @action_metric_name = name
-        end
-      end
-
-      class SlowSql
-        attr_reader :statement
-        attr_reader :metric_name
-        attr_reader :duration
-        attr_reader :backtrace
-        attr_reader :start_time
-
-
-        def initialize(statement, metric_name, duration, t0,  backtrace=nil)
-          @start_time = t0
-          @statement = statement
-          @metric_name = metric_name
-          @duration = duration
-          @backtrace = backtrace
-        end
-
-        def sql
-          statement.sql
-        end
-
-        def obfuscate
-          TingYun::Agent::Database.obfuscate_sql(statement)
-        end
-
-
-        def normalize
-          TingYun::Agent::Database::Obfuscator.instance.default_sql_obfuscator(statement)
-        end
-
-        def explain
-          if statement.config && statement.explainer
-            TingYun::Agent::Database.explain_sql(statement.sql, statement.config, statement.explainer)
-          end
-        end
-
-        # We can't serialize the explainer, so clear it before we transmit
-        def prepare_to_send
-          statement.explainer = nil
-        end
       end
 
-
-
-      class SqlTrace < TingYun::Metrics::Stats
-
-        attr_reader :action_metric_name
-        attr_reader :uri
-        attr_reader :sql
-        attr_reader :slow_sql
-        attr_reader :params
-
-        def initialize(normalized_query, slow_sql, action_name, uri)
-          super()
-          @params = {}
-
-          @action_metric_name = action_name
-          @slow_sql = slow_sql
-          @sql = normalized_query
-          @uri = uri
-          @params[:stacktrace] = slow_sql.backtrace
-          record_data_point(float(slow_sql.duration))
-        end
-
-        def aggregate(slow_sql, action_name, uri)
-          if slow_sql.duration > max_call_time
-            @action_metric_name = action_name
-            @slow_sql = slow_sql
-            @uri = uri
-            @params[:stacktrace] = slow_sql.backtrace
-          end
-          record_data_point(float(slow_sql.duration))
-        end
-
-        def aggregate_trace(trace)
-          aggregate(trace.slow_sql, trace.path, trace.url)
-        end
-
-        def prepare_to_send
-          @sql = @slow_sql.sql unless need_to_obfuscate?
-          @params[:explainPlan] = @slow_sql.explain if need_to_explain?
-        end
-
-        def need_to_obfuscate?
-          Agent.config[:'nbs.action_tracer.record_sql'].to_s == 'obfuscated'
-        end
-
-        def need_to_explain?
-          Agent.config[:'nbs.action_tracer.explain_enabled'] &&  @slow_sql.duration * 1000 > TingYun::Agent.config[:'nbs.action_tracer.explain_threshold']
-        end
-
-
-        include TingYun::Support::Coerce
-
-        def to_collector_array(encoder)
-          [
-              @slow_sql.start_time,
-              string(@action_metric_name),
-              string(@slow_sql.metric_name),
-              string(@uri||@action_metric_name),
-              string(@sql),
-              int(@call_count),
-              TingYun::Helper.time_to_millis(@total_call_time),
-              TingYun::Helper.time_to_millis(@max_call_time),
-              TingYun::Helper.time_to_millis(@min_call_time),
-              encoder.encode(@params)
-          ]
-        end
-      end
     end
   end
 end