tingyun_rpm 1.6.1 → 3.2.0

data/lib/ting_yun/agent/datastore.rb

@@ -9,8 +9,9 @@ module TingYun
     module Datastore
       def self.wrap(product, operation, collection = nil, ip_address = nil, port = nil, dbname=nil,  callback = nil )
         return yield unless operation
-
-        klass_name, *metrics = TingYun::Agent::Datastore::MetricHelper.metrics_for(product, operation, ip_address , port, dbname,collection )
+        klass_name, *metrics = TingYun::Agent::Datastore::MetricHelper.metrics_for(product, operation, ip_address ,
+                                                                                   port,
+                                                                        dbname,collection )
         TingYun::Agent::MethodTracerHelpers.trace_execution_scoped(metrics, {}, nil, klass_name) do
           t0 = Time.now
           begin
@@ -20,6 +21,16 @@ module TingYun
             if callback
               callback.call(elapsed_time)
             end
+            config = {
+                :product => product,
+                :operation => operation,
+                :database => collection,
+                :host => ip_address,
+                :port => port,
+                :type => product,
+                :nosql => klass_name
+            }
+            ::TingYun::Agent::Collector::TransactionSampler.notice_nosql_statement(config,elapsed_time*1000, :nosql)
           end
         end
       end

data/lib/ting_yun/agent/datastore/metric_helper.rb

@@ -20,19 +20,19 @@ module TingYun
 
         def self.metric_name(product, collection, operation,host,port,dbname)
           if checkNosql(product)
-            return "#{product}/#{host}:#{port}%2F#{dbname}%2F#{collection}/#{operation}" if product=="MongoDB"
-            "#{product}/#{host}:#{port}%2F#{collection}/#{operation}"
+            return "#{product}/#{host}:#{port}/#{dbname}/#{collection}/#{operation}" if product=="MongoDB"
+            "#{product}/#{host}:#{port}/#{collection}/#{operation}"
           else
-            "Database #{product}/#{host}:#{port}%2F#{dbname}%2F#{collection}/#{operation}"
+            "Database #{product}/#{host}:#{port}/#{dbname}/#{collection}/#{operation}"
           end
         end
 
         def self.metric_name_others(product, collection, operation)
           collection ||= 'NULL'
           if checkNosql(product)
-            "#{product}%2F#{collection}/#{operation}"
+            "#{product}/#{collection}/#{operation}"
           else
-            "Database #{product}%2F#{collection}/#{operation}"
+            "Database #{product}/#{collection}/#{operation}"
           end
         end
 
@@ -68,9 +68,10 @@ module TingYun
           end
 
           if checkNosql(product)
-            metrics << (product=="MongoDB" ? "#{product}/#{host}:#{port}%2F#{dbname}/All" : "#{product}/#{host}:#{port}/All")
+            metrics << (product=="MongoDB" ? "#{product}/#{host}:#{port}/#{dbname}/All" :
+                            "#{product}/#{host}:#{port}/All")
           else
-            metrics << "Database #{product}/#{host}:#{port}%2F#{dbname}/All"
+            metrics << "Database #{product}/#{host}:#{port}/#{dbname}/All"
           end
           metrics.unshift metric_name(product, collection, operation,host,port,dbname) if collection
           metrics.unshift  "#{product}/#{host}:#{port}/#{operation}" if product=="Memcached"

data/lib/ting_yun/agent/instance_methods/connect.rb

@@ -7,6 +7,7 @@ require 'ting_yun/agent/instance_methods/handle_errors'
 require 'ting_yun/environment_report'
 
 
+
 module TingYun
   module Agent
     module InstanceMethods
@@ -70,14 +71,23 @@ module TingYun
         def connect_settings
           sanitize_environment_report
           settings = {
-              :pid => $$,
-              :port => ::TingYun::Agent.config[:port],
               :host => local_host,
-              :appName => ::TingYun::Agent.config.app_names,
+              :port => ::TingYun::Agent.config[:port],
+              :instanceName => ::TingYun::Agent.config[:'instance.name'],
+              :appName => ::TingYun::Agent.config[:app_name],
               :language => 'Ruby',
-              :agentVersion => ::TingYun::VERSION::STRING,
-              :env => @environment_report,
-              :config => ::TingYun::Agent.config.to_collector_hash
+              :agentVersion => TingYun::VERSION::STRING,
+              :firstRun  => @connect_state==:first,
+              :oneAgentUuid=> 'oneAgentUuid',
+              :environment => {
+                  :meta => {
+                      :pid => $$,
+                      :agentVersion => ::TingYun::VERSION::STRING,
+                      :readonly => true
+                  },
+                  :env => @environment_report,
+                  :config => ::TingYun::Agent.config.to_collector_hash
+              }
           }
           settings
         end
@@ -108,13 +118,15 @@ module TingYun
           handle_force_restart(e)
           retry
         rescue TingYun::Support::Exception::ServerConnectionException => e
-          handle_delay_restart(e, 60)
+          handle_delay_restart(e, 30)
           retry
         rescue => e
-          handle_delay_restart(e, 60)
+          handle_delay_restart(e, 30)
           retry
         end
 
+
+
         # Takes a hash of configuration data returned from the
         # server and uses it to set local variables and to
         # initialize various parts of the agent that are configured
@@ -151,6 +163,7 @@ module TingYun
           TingYun::Agent.disable_all_tracing { connect!(:keep_retrying => false) }
         end
       end
+
     end
   end
 end

data/lib/ting_yun/agent/instance_methods/container_data_manager.rb

@@ -7,7 +7,6 @@ require 'ting_yun/agent/collector/transaction_sampler'
 
 require 'ting_yun/agent/collector/sql_sampler'
 
-
 module TingYun
   module Agent
     module InstanceMethods
@@ -32,7 +31,7 @@ module TingYun
         def init_containers
           @stats_engine = TingYun::Agent::Collector::StatsEngine.new
           @error_collector = TingYun::Agent::Collector::ErrorCollector.new
-          @transaction_sampler = TingYun::Agent::Collector::TransactionSampler.new
+            @transaction_sampler = TingYun::Agent::Collector::TransactionSampler.new
           @sql_sampler = TingYun::Agent::Collector::SqlSampler.new
         end
 
@@ -49,12 +48,12 @@ module TingYun
 
           @events.notify(:middleware_harvest)
           @service.session do # use http keep-alive
-            harvest_and_send_errors
-            harvest_and_send_external_errors
-            harvest_and_send_exceptions
-            harvest_and_send_timeslice_data
+            # harvest_and_send_errors
+            # harvest_and_send_external_errors
+            # harvest_and_send_exceptions
+            # harvest_and_send_timeslice_data
             harvest_and_send_transaction_traces
-            harvest_and_send_slowest_sql
+            # harvest_and_send_slowest_sql
           end
         end
 
@@ -123,7 +122,7 @@ module TingYun
           items
         end
 
-        def send_data_to_endpoint(endpoint, items, container)
+        def send_data_to_endpointV2(endpoint, items, container)
           TingYun::Agent.logger.info("Sending #{items.size} items to #{endpoint}")
           begin
             if container.respond_to?(:harvest_base_quantile_hash!)
@@ -142,6 +141,19 @@ module TingYun
 
         end
 
+        def send_data_to_endpoint(endpoint, items, container)
+          TingYun::Agent.logger.info("Sending #{items.size} items to #{endpoint}")
+          begin
+            @service.send(endpoint, items)
+          rescue => e
+            TingYun::Agent.logger.info("Unable to send #{endpoint} data, will try again later. Error: ", e)
+            # container.merge!(items)
+            raise
+          ensure
+            items = nil #  to GC
+          end
+        end
+
       end
 
     end

data/lib/ting_yun/agent/instance_methods/start.rb

@@ -12,10 +12,10 @@ module TingYun
         # Check to see if the agent should start, returning +true+ if it should.
         # should hava the vaild app_name, unstart-state and able to start
         # The agent is disabled when it is not force enabled by the
-        # 'nbs.agent_enabled' option (e.g. in a manual start), or
+        # 'agent_enabled' option (e.g. in a manual start), or
         # enabled normally through the configuration file
         def agent_should_start?
-          return false if already_started? || !TingYun::Agent.config[:'nbs.agent_enabled']
+          return false if already_started? || !TingYun::Agent.config[:'agent_enabled']
           unless app_name_configured?
             TingYun::Agent.logger.error "No application name configured.",
                                "The Agent cannot start without at least one. Please check your ",
@@ -136,7 +136,7 @@ module TingYun
           end
 
           return if !needs_restart ||
-              !Agent.config[:'nbs.agent_enabled'] || disconnected?
+              !Agent.config[:'agent_enabled'] || disconnected?
 
           ::TingYun::Agent.logger.debug "Starting the worker thread in #{Process.pid} (parent #{Process.ppid}) after forking."
 

data/lib/ting_yun/agent/instance_methods/start_worker_thread.rb

@@ -37,17 +37,22 @@ module TingYun
 
         def create_and_run_event_loop
           @event_loop = TingYun::Agent::Event::EventLoop.new
-
-          @event_loop.on(:report_data) do
-            transmit_data
+          @event_loop.on(:getCmd) do
+             @service.getCmd
           end
-          @event_loop.fire_every(Agent.config[:data_report_period], :report_data)
+          @event_loop.fire_every(30, :getCmd)
+          #
+          # @event_loop.on(:report_data) do
+          #   transmit_data
+          # end
+          # @event_loop.fire_every(Agent.config[:data_report_period], :report_data)
 
           @event_loop.on(:create_new_logfile) do
             TingYun::Logger::CreateLoggerHelper.create_new_logfile
           end
           @event_loop.fire_every(TingYun::Agent.config[:agent_log_file_check_days]*60*60*24, :create_new_logfile)
 
+
           @event_loop.run
         end
       end

data/lib/ting_yun/agent/method_tracer_helpers.rb

@@ -22,7 +22,8 @@ module TingYun
         end
       end
 
-      def trace_execution_scoped_footer(state, t0, first_name, metric_names, expected_frame, options, t1=Time.now.to_f, klass_name=nil, error = nil)
+      def trace_execution_scoped_footerV2(state, t0, first_name, metric_names, expected_frame, options, t1=Time.now
+                                                                                                              .to_f, klass_name=nil, error = nil)
         log_errors(:trace_execution_scoped_footer) do
           if expected_frame
             stack = state.traced_method_stack
@@ -49,6 +50,33 @@ module TingYun
         end
       end
 
+      def trace_execution_scoped_footer(state, t0, first_name, metric_names, expected_frame, options, t1=Time.now.to_f, klass_name=nil, error = nil)
+        log_errors(:trace_execution_scoped_footer) do
+          if expected_frame
+            stack = state.traced_method_stack
+            create_metrics = options.has_key?(:metric) ? options[:metric] : true
+            frame = stack.pop_frame(state, expected_frame, first_name, t1, create_metrics, klass_name, error)
+
+            if create_metrics
+              duration = (t1 - t0)*1000
+              exclusive = duration - frame.children_time
+              if duration < 0
+                ::TingYun::Agent.logger.log_once(:warn, "metric_duration_negative:#{first_name}",
+                                                 "Metric #{first_name} has negative duration: #{duration} ms")
+              end
+              if exclusive < 0
+                ::TingYun::Agent.logger.log_once(:warn, "metric_exclusive_negative:#{first_name}",
+                                                 "Metric #{first_name} has negative exclusive time: duration = #{duration} ms, child_time = #{frame.children_time}")
+              end
+              # record_metrics(state, first_name, metric_names, duration, exclusive, options)
+              # if first_name.start_with?('WebAction')
+              #   state.current_transaction.base_quantile_hash[first_name] = duration
+              # end
+            end
+          end
+        end
+      end
+
       def record_metrics(state, first_name, other_names, duration, exclusive, options)
         record_scoped_metric = options.has_key?(:scoped_metric) ? options[:scoped_metric] : true
         stat_engine = TingYun::Agent.instance.stats_engine

data/lib/ting_yun/agent/transaction.rb

@@ -25,9 +25,10 @@ module TingYun
 
 
       SUBTRANSACTION_PREFIX = 'Nested/'.freeze
-      CONTROLLER_PREFIX = 'WebAction/'.freeze
-      BACKGROUND_PREFIX = 'BackgroundAction/'.freeze
-      RAKE_TRANSACTION_PREFIX     = 'BackgroundAction/Rake'.freeze
+      CONTROLLER_PREFIX = 'Transaction/'.freeze
+      COSS_CONTROLLER_PREFIX = 'WebAction/'.freeze
+      BACKGROUND_PREFIX = 'TaskAction/'.freeze
+      RAKE_TRANSACTION_PREFIX     = 'TaskAction/Rake'.freeze
       TASK_PREFIX = 'OtherTransaction/Background/'.freeze
       RACK_PREFIX = 'Rack/'.freeze
       SINATRA_PREFIX = 'WebAction/Sinatra/'.freeze
@@ -67,7 +68,6 @@ module TingYun
 
       def initialize(category, client_transaction_id, options)
         @start_time = Time.now.to_f
-
         @exceptions = TingYun::Agent::Transaction::Exceptions.new
         @metrics = TingYun::Agent::TransactionMetrics.new
         @attributes = TingYun::Agent::Transaction::Attributes.new
@@ -85,6 +85,7 @@ module TingYun
 
         if request = options[:request]
           @request_attributes = TingYun::Agent::Transaction::RequestAttributes.new request
+          @attributes.merge_request_header(@request_attributes.header.select{|k,v|k.start_with?("HTTP")})
         else
           @request_attributes = nil
         end
@@ -118,7 +119,6 @@ module TingYun
 
 
       def stop(state, end_time, outermost_frame, summary_metrics = [])
-
         freeze_name_and_execute
 
         if @has_children or @is_mq
@@ -149,7 +149,7 @@ module TingYun
       end
 
 
-      def commit(state, end_time, outermost_node_name)
+      def commitV2(state, end_time, outermost_node_name)
 
         assign_agent_attributes(state)
 
@@ -167,6 +167,24 @@ module TingYun
         TingYun::Agent.instance.stats_engine.record_base_quantile(@base_quantile_hash) unless @exceptions.had_error?
       end
 
+      def commit(state, end_time, outermost_node_name)
+
+        assign_agent_attributes(state)
+
+
+        TingYun::Agent.instance.transaction_sampler.on_finishing_transaction(state, self, end_time,@exceptions)
+
+        TingYun::Agent.instance.sql_sampler.on_finishing_transaction(state, @frozen_name)
+        #
+        # record_summary_metrics(state, outermost_node_name, end_time) unless @exceptions.had_error?
+        # @apdex.record_apdex(@frozen_name, end_time, @exceptions.had_error?)
+        # @exceptions.record_exceptions(@attributes)
+        #
+        #
+        # TingYun::Agent.instance.stats_engine.merge_transaction_metrics!(@metrics, best_name)
+        # TingYun::Agent.instance.stats_engine.record_base_quantile(@base_quantile_hash) unless @exceptions.had_error?
+      end
+
     end
   end
 end

data/lib/ting_yun/agent/transaction/attributes.rb

@@ -5,11 +5,14 @@ module TingYun
     class Transaction
       class Attributes
 
-        attr_accessor :agent_attributes, :request_params, :custom_params
+        attr_accessor :agent_attributes, :request_params, :custom_params, :response_header,:request_header,:user_info
         def initialize
           @agent_attributes  = {:httpStatus => 0} #defaul value
           @request_params = {}
-          @custom_params = {}
+          @custom_params = {:threadId => $$}
+          @response_header = {}
+          @request_header = {}
+          @user_info = {}
         end
 
         # no longer to care about the value if nil or not
@@ -24,6 +27,18 @@ module TingYun
         def add_custom_params(key, value)
           @custom_params[key] = value
         end
+
+        def add_request_params(key, value)
+          @request_header[key] = value
+        end
+
+        def merge_request_header(hash)
+          @request_header.merge!(hash) if hash
+        end
+
+        def add_user_info(key, value)
+          @user_info[key] = value
+        end
       end
     end
   end

data/lib/ting_yun/agent/transaction/class_method.rb

@@ -48,6 +48,7 @@ module TingYun
           if txn.frame_stack.empty?
             txn.stop(state, end_time, nested_frame, summary_metric_names)
             state.reset
+            TingYun::Agent.upload
           else
             nested_name = Transaction.nested_transaction_name nested_frame.name
 

data/lib/ting_yun/agent/transaction/instance_method.rb

@@ -14,15 +14,14 @@ module TingYun
         end
 
         def create_nested_frame(state, category, options)
-
           if options[:filtered_params] && !options[:filtered_params].empty?
             attributes.merge_request_parameters(options[:filtered_params])
           end
+          state.add_current_node_params(:method => options[:method]) unless options[:method].nil?
           @has_children = true
           frame_stack.push TingYun::Agent::MethodTracerHelpers.trace_execution_scoped_header(state, Time.now.to_f)
           name_last_frame(options[:transaction_name])
-
-          set_default_transaction_name(options[:transaction_name], category)
+          set_default_transaction_name("#{state.action_type}/#{options[:transaction_name]}", category)
         end
 
 
@@ -89,11 +88,15 @@ module TingYun
 
           if @request_attributes
             @request_attributes.assign_agent_attributes @attributes
+            @request_attributes.assign_user_info_attributes @attributes
           end
 
           @attributes.add_agent_attribute(:tx_id,  @guid);
+          @attributes.add_agent_attribute(:tmd5,  state.transaction_name_md5);
           @attributes.add_agent_attribute(:metric_name,  best_name);
           @attributes.add_agent_attribute(:trace_id, state.trace_id || "0")
+          @attributes.add_agent_attribute(:refid, state.extenel_req_id)
+          @attributes.add_agent_attribute(:cross, state.call_list)
         end
 
 
@@ -135,7 +138,10 @@ module TingYun
           unless @frozen_name
             @frozen_name = best_name
           end
-          @frozen_name = CONTROLLER_PREFIX + @frozen_name unless @frozen_name.start_with? CONTROLLER_PREFIX,BACKGROUND_PREFIX
+          state = TingYun::Agent::TransactionState.tl_get
+          unless @frozen_name.start_with? CONTROLLER_PREFIX, BACKGROUND_PREFIX, COSS_CONTROLLER_PREFIX
+            @frozen_name = state.action_type + @frozen_name
+          end
           yield if block_given?
         end
 

data/lib/ting_yun/agent/transaction/request_attributes.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'ting_yun/support/http_clients/uri_util'
+require 'ting_yun/support/serialize/json_wrapper'
 
 module TingYun
   module Agent
@@ -8,7 +9,7 @@ module TingYun
       class RequestAttributes
 
         attr_reader :request_path, :referer, :accept, :content_length, :host,
-                    :port, :user_agent, :request_method, :header, :cookie
+                    :port, :user_agent, :request_method, :header, :cookie, :ip, :url, :params,:queryString
 
         HTTP_ACCEPT_HEADER_KEY = 'HTTP_ACCEPT'.freeze
 
@@ -23,17 +24,55 @@ module TingYun
           @user_agent = attribute_from_request request, :user_agent
           @request_method = attribute_from_request request, :request_method
           @cookie = set_cookie(request)
+          @ip = attribute_from_env(request, 'HTTP_X_FORWARDED_FOR')|| request.ip
+          @url = request.scheme + "://"+ request.host_with_port +  @request_path
+          @params =  request.params
+          @queryString = request.query_string
         end
 
         def assign_agent_attributes(attributes)
           attributes.add_agent_attribute :request_path, request_path
-          attributes.add_agent_attribute :referer, referer
+          attributes.add_agent_attribute :referer, referer  unless referer.nil?
           attributes.add_agent_attribute :accept, accept
-          attributes.add_agent_attribute :contentLength, content_length
+          attributes.add_agent_attribute :contentLength, content_length  unless content_length.nil?
           attributes.add_agent_attribute :host, host
           attributes.add_agent_attribute :port, port
           attributes.add_agent_attribute :userAgent, user_agent
           attributes.add_agent_attribute :method, request_method
+          attributes.add_agent_attribute :ip, ip
+          attributes.add_agent_attribute :url, url
+          attributes.add_agent_attribute :queryString, queryString
+          attributes.response_header.merge! attributes.agent_attributes
+          attributes.add_agent_attribute :params, params
+          attributes.add_agent_attribute :cookie, @cookie
+        end
+
+        def assign_user_info_attributes(attributes)
+          rule = TingYun::Support::Serialize::JSONWrapper.load(TingYun::Agent.config[:'user_info.rules'])
+          rule["list"].each do |r|
+            if [2,9].include?(r["type"]) && !@params.nil?
+              attributes.add_user_info("origin"=> 1)
+              attributes.add_user_info("value"=> @params[r["value"]])
+            elsif [7,10].include?(r["type"]) && !@cookie.nil?
+              attributes.add_user_info("origin"=> 1)
+              attributes.add_user_info("value"=> @cookie[r["value"]])
+            end
+          end
+        end
+
+        def assign_request_attributes(attributes)
+          attributes.add_request_params :request_path, request_path
+          attributes.add_request_params :referer, referer unless referer.nil?
+          attributes.add_request_params :accept, accept
+          attributes.add_request_params(:contentLength, content_length) unless content_length.nil?
+          attributes.add_request_params :host, host
+          attributes.add_request_params :port, port
+          attributes.add_request_params :userAgent, user_agent
+          attributes.add_request_params :method, request_method
+          attributes.add_request_params :ip, ip
+          attributes.add_request_params :url, url
+          attributes.add_request_params :params, params
+          attributes.add_request_params :queryString, queryString
         end