tingyun_rpm 1.0.6

data/lib/ting_yun/agent/collector/transaction_sampler.rb

@@ -0,0 +1,226 @@
+# encoding: utf-8
+
+require 'ting_yun/agent/transaction/transaction_sample_builder'
+require 'ting_yun/agent/collector/transaction_sampler/slowest_sample_buffer'
+
+require 'ting_yun/agent/transaction/transaction_state'
+
+
+module TingYun
+  module Agent
+    module Collector
+      class TransactionSampler
+
+
+        attr_accessor :last_sample
+
+
+        def initialize
+          @lock = Mutex.new
+          @sample_buffers = []
+          @sample_buffers << TingYun::Agent::Collector::TransactionSampler::SlowestSampleBuffer.new
+        end
+
+
+        def notice_push_frame(state, time=Time.now)
+          builder = state.transaction_sample_builder
+          return unless builder
+          builder.trace_entry(time.to_f)
+        end
+
+        # Informs the transaction sample builder about the end of a traced frame
+        def notice_pop_frame(state, frame, time = Time.now)
+          builder = state.transaction_sample_builder
+          return unless builder
+          builder.trace_exit(frame, time.to_f)
+        end
+
+        def enabled?
+          TingYun::Agent.config[:'nbs.action_tracer.enabled']
+        end
+
+        def on_start_transaction(state, time)
+          start_builder(state, time)
+        end
+
+        def start_builder(state, time=nil)
+          if enabled?
+            state.transaction_sample_builder ||= TingYun::Agent::TransactionSampleBuilder.new(time)
+          else
+            state.transaction_sample_builder = nil
+          end
+        end
+
+
+        def on_finishing_transaction(state, txn, time=Time.now.to_f)
+          last_builder = state.transaction_sample_builder
+          return unless last_builder && enabled?
+
+          last_builder.finish_trace(time)
+
+          state.transaction_sample_builder = nil
+
+
+          final_trace = last_builder.trace
+          final_trace.metric_name = txn.best_name
+          final_trace.uri = txn.request_path
+          final_trace.attributes = txn.attributes
+
+
+          @lock.synchronize do
+            @last_sample = final_trace
+
+            store_sample(@last_sample)
+            @last_sample
+          end
+        end
+
+        def store_sample(sample)
+          @sample_buffers.each do |sample_buffer|
+            sample_buffer.store(sample)
+          end
+        end
+
+
+        # Attaches an SQL query on the current transaction trace node.
+        # @param sql [String] the SQL query being recorded
+        # @param config [Object] the driver configuration for the connection
+        # @param duration [Float] number of seconds the query took to execute
+        # @param explainer [Proc] for internal use only - 3rd-party clients must
+        #                         not pass this parameter.
+        # duration{:type => sec}
+        def notice_sql(sql, config, duration, state=nil, explainer=nil)
+          # some statements (particularly INSERTS with large BLOBS
+          # may be very large; we should trim them to a maximum usable length
+          state ||= TingYun::Agent::TransactionState.tl_get
+          builder = state.transaction_sample_builder
+          if state.sql_recorded?
+            statement = TingYun::Agent::Database::Statement.new(sql, config, explainer)
+            action_tracer_segment(builder, statement, duration, :sql)
+          end
+        end
+
+        # duration{:type => sec}
+        def notice_nosql(key, duration) #THREAD_LOCAL_ACCESS
+          builder = tl_builder
+          return unless builder
+          action_tracer_segment(builder, key, duration, :key)
+        end
+
+        # duration{:type => sec}
+        def notice_nosql_statement(statement, duration) #THREAD_LOCAL_ACCESS
+          builder = tl_builder
+          return unless builder
+          action_tracer_segment(builder, statement, duration, :statement)
+        end
+
+
+
+        MAX_DATA_LENGTH = 16384
+        # This method is used to record metadata into the currently
+        # active node like a sql query, memcache key, or Net::HTTP uri
+        #
+        # duration is milliseconds, float value.
+        # duration{:type => sec}
+        def action_tracer_segment(builder, message, duration, key)
+          return unless builder
+          node = builder.current_node
+          if node
+            if key == :sql
+              statement = node[:sql]
+              if statement && !statement.sql.empty?
+                statement.sql = self.class.truncate_message(statement.sql + "\n#{message.sql}") if statement.sql.length <= MAX_DATA_LENGTH
+              else
+                # message is expected to have been pre-truncated by notice_sql
+                node[:sql] =  message
+              end
+            else
+              node[key] = self.class.truncate_message(message)
+            end
+            append_backtrace(node, duration)
+          end
+        end
+
+        # Truncates the message to `MAX_DATA_LENGTH` if needed, and
+        # appends an ellipsis because it makes the trucation clearer in
+        # the UI
+        def self.truncate_message(message)
+          if message.length > (MAX_DATA_LENGTH - 4)
+            message.slice!(MAX_DATA_LENGTH - 4..message.length)
+            message << '...'
+          else
+            message
+          end
+        end
+
+        # Appends a backtrace to a node if that node took longer
+        # than the specified duration
+        def append_backtrace(node, duration)
+          if duration*1000 >= Agent.config[:'nbs.action_tracer.stack_trace_threshold']
+            node[:stacktrace] = caller.reject! { |t| t.include?('tingyun_rpm') }
+          end
+
+        end
+
+        def harvest!
+          return [] unless enabled?
+
+          samples = @lock.synchronize do
+            @last_sample = nil
+            harvest_from_sample_buffers
+          end
+
+          prepare_samples(samples)
+        end
+
+        def harvest_from_sample_buffers
+          result = []
+          @sample_buffers.each { |buffer| result.concat(buffer.harvest_samples) }
+          result.uniq
+        end
+
+        def prepare_samples(samples)
+          samples.select do |sample|
+            begin
+              sample.prepare_to_send!
+            rescue => e
+
+              TingYun::Agent.logger.error('Failed to prepare transaction trace. Error: ', e)
+
+              false
+            else
+              true
+            end
+          end
+        end
+
+        def merge!
+
+          @lock.synchronize do
+            @sample_buffers.each do |buffer|
+              buffer.store_previous(previous)
+            end
+          end
+        end
+
+        def reset!
+          @lock.synchronize do
+            @sample_buffers.each { |sample| sample.reset! }
+          end
+        end
+
+
+        def add_node_info(params)
+          builder = tl_builder
+          return unless builder
+          params.each { |k,v| builder.current_node.instance_variable_set(('@'<<k.to_s).to_sym, v)  }
+        end
+
+        def tl_builder
+          TingYun::Agent::TransactionState.tl_get.transaction_sample_builder
+        end
+
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/container_data_manager.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+require 'ting_yun/support/exception'
+require 'ting_yun/agent/collector/stats_engine'
+require 'ting_yun/agent/collector/error_collector'
+
+module TingYun
+  module Agent
+    module ContainerDataManager
+
+      attr_reader :stats_engine, :error_collector, :transaction_sampler
+
+      def drop_buffered_data
+        @stats_engine.reset!
+      end
+
+      # private
+
+      def init_containers
+        @stats_engine = TingYun::Agent::Collector::StatsEngine.new
+        @error_collector = TingYun::Agent::Collector::ErrorCollector.new
+      end
+
+      def container_for_endpoint(endpoint)
+        case endpoint
+          when :metric_data             then @stats_engine
+            # type code here
+        end
+      end
+
+      def transmit_data
+        ::TingYun::Agent.logger.debug('Sending data to Ting Yun Service')
+
+        @service.session do # use http keep-alive
+          harvest_and_send_errors
+          harvest_and_send_timeslice_data
+        end
+      end
+
+      def harvest_and_send_timeslice_data
+        harvest_and_send_from_container(@stats_engine, :metric_data)
+      end
+
+      def harvest_and_send_errors
+        harvest_and_send_from_container(@error_collector.error_trace_array, :error_data)
+      end
+
+      # Harvests data from the given container, sends it to the named endpoint
+      # on the service, and automatically merges back in upon a recoverable
+      # failure.
+      #
+      # The given container should respond to:
+      #
+      #  #harvest!
+      #    returns an enumerable collection of data items to be sent to the
+      #    collector.
+      #
+      #  #reset!
+      #    drop any stored data and reset to a clean state.
+      #
+      #  #merge!(items)
+      #    merge the given items back into the internal buffer of the
+      #    container, so that they may be harvested again later.
+      #
+      def harvest_and_send_from_container(container,endpoint)
+        items = harvest_from_container(container, endpoint)
+        if !items.empty? && TingYun::Agent.config[:'nbs.agent_enabled']
+          send_data_to_endpoint(endpoint, items, container)
+        end
+      end
+
+      def harvest_from_container(container, endpoint)
+        items =[]
+        begin
+          items = container.harvest!
+        rescue => e
+          TingYun::Agent.logger.error("Failed to harvest #{endpoint} data, resetting. Error: ", e)
+          container.reset!
+        end
+        items
+      end
+
+      def send_data_to_endpoint(endpoint, items, container)
+        TingYun::Agent.logger.debug("Sending #{items.size} items to #{endpoint}")
+        begin
+          @service.send(endpoint, items)
+        rescue => e
+          TingYun::Agent.logger.info("Unable to send #{endpoint} data, will try again later. Error: ", e)
+        end
+
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/cross_app/cross_app_monitor.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+require 'ting_yun/agent/cross_app/inbound_request_monitor'
+require 'ting_yun/agent/cross_app/cross_app_tracing'
+require 'ting_yun/agent/transaction/transaction_state'
+require 'ting_yun/agent'
+require 'ting_yun/support/serialize/json_wrapper'
+
+
+module TingYun
+  module Agent
+    class CrossAppMonitor < TingYun::Agent::InboundRequestMonitor
+
+
+      TY_ID_HEADER = 'HTTP_X_TINGYUN_ID'.freeze
+      TY_DATA_HEADER = 'X-Tingyun-Tx-Data'.freeze
+
+
+      def on_finished_configuring(events)
+        register_event_listeners(events)
+      end
+
+      # Expected sequence of events:
+      #   :before_call will save our cross application request id to the thread
+      #   :after_call will write our response headers/metrics and clean up the thread
+      def register_event_listeners(events)
+        TingYun::Agent.logger.debug("Wiring up Cross Application Tracing to events after finished configuring")
+
+        events.subscribe(:before_call) do |env| #THREAD_LOCAL_ACCESS
+          if cross_app_enabled?
+            state = TingYun::Agent::TransactionState.tl_get
+            save_referring_transaction_info(state, env)  unless env[TY_ID_HEADER].nil?
+          end
+        end
+
+        events.subscribe(:after_call) do |_status_code, headers, _body| #THREAD_LOCAL_ACCESS
+          state = TingYun::Agent::TransactionState.tl_get
+          state.queue_duration = state.current_transaction.queue_time * 1000
+          state.web_duration = (Time.now - state.current_transaction.start_time) * 1000
+          insert_response_header(state, headers)
+        end
+
+      end
+
+
+      def cross_app_enabled?
+        TingYun::Agent::CrossAppTracing.cross_app_enabled?
+      end
+
+
+      def save_referring_transaction_info(state,request)
+
+        info = request[TY_ID_HEADER].split(';')
+        tingyun_id_secret = info[0]
+        client_transaction_id = info.find do |e|
+          e.match(/x=/)
+        end.split('=')[1] rescue nil
+        client_req_id = info.find do |e|
+          e.match(/r=/)
+        end.split('=')[1] rescue nil
+
+        state.client_tingyun_id_secret = tingyun_id_secret
+        state.client_transaction_id = client_transaction_id
+        state.transaction_sample_builder.trace.tx_id = client_transaction_id
+        state.client_req_id = client_req_id
+      end
+
+
+      def insert_response_header(state, response_headers)
+        if same_account?(state)
+          txn = state.current_transaction
+          unless txn.nil?
+            set_response_headers(state, response_headers)
+          end
+          clear_client_tingyun_id_secret(state)
+        end
+      end
+
+      def clear_client_tingyun_id_secret(state)
+        state.client_tingyun_id_secret = nil
+      end
+
+      def same_account?(state)
+        server_info = TingYun::Agent.config[:tingyunIdSecret].split('|')
+        client_info = (state.client_tingyun_id_secret || '').split('|')
+        if !server_info[0].nil? && server_info[0] == client_info[0] && !server_info[0].empty?
+          return true
+        else
+          return false
+        end
+      end
+
+      def set_response_headers(state, response_headers)
+        response_headers[TY_DATA_HEADER] = TingYun::Support::Serialize::JSONWrapper.dump build_payload(state)
+        TingYun::Agent.logger.debug("now,cross app will send response_headers  #{response_headers[TY_DATA_HEADER]}")
+      end
+
+      def build_payload(state)
+        payload = {
+            :id => TingYun::Agent.config[:tingyunIdSecret].split('|')[1],
+            :action => state.current_transaction.best_name,
+            :trId => state.transaction_sample_builder.trace.guid,
+            :time => {
+                :duration => state.web_duration,
+                :qu => state.queue_duration,
+                :db => state.sql_duration,
+                :ex => state.external_duration,
+                :rds => state.rds_duration,
+                :mc => state.mc_duration,
+                :mon => state.mon_duration,
+                :code => execute_duration(state)
+            }
+        }
+        payload[:tr] = 1 if slow_action_tracer?(state)
+        payload[:r] = state.client_req_id unless state.client_req_id.nil?
+        payload
+      end
+
+      def slow_action_tracer?(state)
+        if state.web_duration > TingYun::Agent.config[:'nbs.action_tracer.action_threshold']
+          return true
+        else
+          return false
+        end
+      end
+
+      def execute_duration(state)
+        state.web_duration - state.queue_duration - state.sql_duration - state.external_duration - state.rds_duration - state.mc_duration - state.mon_duration
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/cross_app/cross_app_tracing.rb

@@ -0,0 +1,202 @@
+# encoding: utf-8
+
+require 'ting_yun/agent/transaction/transaction_state'
+require 'ting_yun/agent/transaction'
+require 'ting_yun/support/http_clients/uri_util'
+require 'ting_yun/support/serialize/json_wrapper'
+require 'ting_yun/instrumentation/support/external_error'
+
+
+module TingYun
+  module Agent
+    module CrossAppTracing
+
+      extend ::TingYun::Instrumentation::Support::ExternalError
+
+      # Exception raised if there is a problem with cross app transactions.
+      class Error < RuntimeError; end
+
+      # The cross app id header for "outgoing" calls
+
+      TY_ID_HEADER = 'X-Tingyun-Id'.freeze
+      TY_DATA_HEADER = 'X-Tingyun-Tx-Data'.freeze
+
+
+
+
+      module_function
+
+
+
+      def tl_trace_http_request(request)
+        t0 = Time.now.to_f
+        state = TingYun::Agent::TransactionState.tl_get
+        return yield unless state.execution_traced?
+
+        begin
+          node = start_trace(state, t0, request)
+          response = yield
+          capture_exception(response,request,'net%2Fhttp')
+        rescue => e
+          klass = "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
+          handle_error(e,klass)
+        ensure
+          finish_trace(state, t0, node, request, response)
+        end
+        return response
+      end
+
+      def start_trace(state, t0, request)
+        # inject_request_headers(state, request) if cross_app_enabled?
+        stack = state.traced_method_stack
+        node = stack.push_frame(state,:http_request,t0)
+
+        return node
+      end
+
+      def finish_trace(state, t0, node, request, response)
+        t1 = Time.now.to_f
+        duration = (t1- t0) * 1000
+        state.external_duration = duration
+
+        begin
+          if request
+            metrics = metrics_for(state, request, response)
+            node_name = metrics.pop
+            scoped_metric = metrics.pop
+
+            stats_engine.record_scoped_and_unscoped_metrics(state, scoped_metric, metrics, duration)
+
+            if node
+              node.name = node_name
+              add_transaction_trace_info(request, response)
+            end
+          end
+        ensure
+          if node
+            stack = state.traced_method_stack
+            stack.pop_frame(state, node, node_name, t1)
+          end
+        end
+      rescue => err
+        TingYun::Agent.logger.error "Uncaught exception while finishing an HTTP request trace", err
+
+      end
+
+
+
+      def add_transaction_trace_info(request, response)
+        state = TingYun::Agent::TransactionState.tl_get
+        filtered_uri = TingYun::Agent::HTTPClients::URIUtil.filter_uri request.uri
+        transaction_sampler.add_node_info(:uri => filtered_uri)
+        if response && response_is_cross_app?( response )
+          transaction_sampler.tl_builder.current_node[:txId] = state.client_transaction_id || state.request_guid
+          my_data = TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
+          transaction_sampler.tl_builder.current_node[:txData] = my_data
+        end
+      end
+
+      def metrics_for(state, request, response)
+        metrics = common_metrics(request)
+
+        if response && response_is_cross_app?( response )
+          begin
+            metrics.concat metrics_for_cross_app_response( request, response )
+          rescue => err
+            # Fall back to regular metrics if there's a problem with x-process metrics
+            TingYun::Agent.logger.debug "%p while fetching x-process metrics: %s" %
+                                            [ err.class, err.message ]
+            metrics.concat metrics_for_regular_request( request )
+          end
+        else
+          metrics.concat metrics_for_regular_request( request )
+        end
+
+        return metrics
+      end
+
+      def common_metrics(request)
+        metrics = [ "External/NULL/ALL" ]
+
+        if TingYun::Agent::Transaction.recording_web_transaction?
+          metrics << "External/NULL/AllWeb"
+        else
+          metrics << "External/NULL/AllBackground"
+        end
+
+        return metrics
+      end
+
+      def metrics_for_regular_request( request )
+        state = TingYun::Agent::TransactionState.tl_get
+        metrics = []
+        metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
+        metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
+
+        return metrics
+      end
+
+      def stats_engine
+        ::TingYun::Agent.instance.stats_engine
+      end
+
+      def transaction_sampler
+        ::TingYun::Agent.instance.transaction_sampler
+      end
+
+
+      def cross_app_enabled?
+        valid_tingyun_secret_id? && web_action_tracer_enabled? && cross_application_tracer_enabled?
+      end
+
+      def web_action_tracer_enabled?
+        TingYun::Agent.config[:'nbs.action_tracer.enabled']
+      end
+
+      def cross_application_tracer_enabled?
+        TingYun::Agent.config[:'nbs.transaction_tracer.enabled']
+      end
+
+      def valid_tingyun_secret_id?
+        TingYun::Agent.config[:tingyunIdSecret] && TingYun::Agent.config[:tingyunIdSecret].size > 0
+      end
+
+      # Inject the X-Process header into the outgoing +request+.
+      def inject_request_headers(state, request)
+        cross_app_id  = TingYun::Agent.config[:tingyunIdSecret] or
+            raise TingYun::Agent::CrossAppTracing::Error, "no tingyunIdSecret configured"
+
+        txn_guid =  state.client_transaction_id || state.request_guid
+        state.transaction_sample_builder.trace.tx_id = txn_guid
+        request[TY_ID_HEADER] = "#{cross_app_id};c=1;x=#{txn_guid}"
+
+      rescue TingYun::Agent::CrossAppTracing::Error => err
+        TingYun::Agent.logger.debug "Not injecting x-process header", err
+      end
+
+      # Returns +true+ if Cross Application Tracing is enabled, and the given +response+
+      # has the appropriate headers.
+      def response_is_cross_app?( response )
+        return false unless cross_app_enabled?
+        unless response[TY_DATA_HEADER]
+          return false
+        end
+        return true
+      end
+
+      # Return the set of metric objects appropriate for the given cross app
+      # +response+.
+      def metrics_for_cross_app_response(request, response )
+        state = TingYun::Agent::TransactionState.tl_get
+        my_data =  TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
+        uri = "#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
+        metrics = []
+        metrics << "cross_app;#{my_data["id"]};#{my_data["action"]};#{uri}"
+        metrics << "External/#{my_data["action"]}:#{uri}"
+
+        return metrics
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/cross_app/inbound_request_monitor.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+# This class serves as the base for objects wanting to monitor and respond to
+# incoming web requests. Examples include cross application tracing and
+# synthetics.
+#
+# Subclasses are expected to define on_finished_configuring(events) which will
+# be called when the agent is fully configured. That method is expected to
+# subscribe to the necessary request events, such as before_call and after_call
+# for the monitor to do its work.
+
+module TingYun
+  module Agent
+    class InboundRequestMonitor
+      def initialize(events)
+        events.subscribe(:finished_configuring) do
+          on_finished_configuring(events)
+        end
+      end
+    end
+  end
+end