tingyun_rpm 1.0.6

data/lib/ting_yun/instrumentation/support/queue_time.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+module TingYun
+  module Instrumentation
+    module Support
+
+      module QueueTime
+        unless defined?(REQUEST_START_HEADER)
+          REQUEST_START_HEADER = 'HTTP_X_REQUEST_START'.freeze
+          QUEUE_START_HEADER = 'HTTP_X_QUEUE_START'.freeze
+          MIDDLEWARE_START_HEADER = 'HTTP_X_MIDDLEWARE_START'.freeze
+          ALL_QUEUE_METRIC = 'WebFrontend/QueueTime'.freeze
+          # any timestamps before this are thrown out and the parser
+          # will try again with a larger unit (2000/1/1 UTC)
+          EARLIEST_ACCEPTABLE_TIME = Time.at(946684800)
+
+          CANDIDATE_HEADERS = [
+              REQUEST_START_HEADER,
+              QUEUE_START_HEADER,
+              MIDDLEWARE_START_HEADER
+          ].freeze
+
+          DIVISORS = [1_000_000, 1_000, 1]
+        end
+
+        module_function
+
+        def parse_frontend_timestamp(headers, now=Time.now)
+          earliest = nil
+
+          CANDIDATE_HEADERS.each do |header|
+            if headers[header]
+              parsed = parse_timestamp(timestamp_string_from_header_value(headers[header]))
+              if parsed && (!earliest || parsed < earliest)
+                earliest = parsed
+              end
+            end
+          end
+
+          if earliest && earliest > now
+            TingYun::Agent.logger.debug("Negative queue time detected, treating as zero: start=#{earliest.to_f} > now=#{now.to_f}")
+            earliest = now
+          end
+
+          earliest
+        end
+
+        def timestamp_string_from_header_value(value)
+          case value
+            when /^\s*([\d+\.]+)\s*$/ then
+              $1
+            # following regexp intentionally unanchored to handle
+            # (ie ignore) leading server names
+            when /t=([\d+\.]+)/ then
+              $1
+          end
+        end
+
+        def parse_timestamp(string)
+          DIVISORS.each do |divisor|
+            begin
+              t = Time.at(string.to_f / divisor)
+              return t if t > EARLIEST_ACCEPTABLE_TIME
+            rescue RangeError
+              # On Ruby versions built with a 32-bit time_t, attempting to
+              # instantiate a Time object in the far future raises a RangeError,
+              # in which case we know we've chosen the wrong divisor.
+            end
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/instrumentation/support/transaction_namer.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+module TingYun
+  module Instrumentation
+    module Support
+      class TransactionNamer
+
+
+        def self.prefix_for_category(txn, category = nil)
+          category ||= (txn && txn.category)
+
+          case category
+            when :controller then
+              ::TingYun::Agent::Transaction::CONTROLLER_PREFIX
+            when :task then
+              ::TingYun::Agent::Transaction::TASK_PREFIX
+            when :rack then
+              ::TingYun::Agent::Transaction::RACK_PREFIX
+            when :uri then
+              ::TingYun::Agent::Transaction::CONTROLLER_PREFIX
+            when :sinatra then
+              ::TingYun::Agent::Transaction::SINATRA_PREFIX
+            when :middleware then
+              ::TingYun::Agent::Transaction::MIDDLEWARE_PREFIX
+            when :grape then
+              ::TingYun::Agent::Transaction::GRAPE_PREFIX
+            when :rake then
+              ::TingYun::Agent::Transaction::RAKE_PREFIX
+            else
+              "#{category.to_s}/" # for internal use only
+          end
+        end
+
+
+        def self.name_for(txn, traced_obj, category, options={})
+          "#{prefix_for_category(txn, category)}#{path_name(traced_obj, options)}"
+        end
+
+        def self.path_name(traced_obj, options={})
+          return options[:path] if options[:path]
+
+          class_name = klass_name(traced_obj, options)
+          if options[:name]
+            if class_name
+              "#{class_name}/#{options[:name]}"
+            else
+              options[:name]
+            end
+          elsif traced_obj.respond_to?(:tingyun_metric_path)
+            traced_obj.tingyun_metric_path
+          else
+            class_name
+          end
+        end
+
+        def self.klass_name(traced_obj, options={})
+          return options[:class_name] if options[:class_name]
+
+          if (traced_obj.is_a?(Class) || traced_obj.is_a?(Module))
+            traced_obj.name
+          else
+            traced_obj.class.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/instrumentation/thrift.rb

@@ -0,0 +1,329 @@
+# encoding: utf-8
+require 'ting_yun/support/helper'
+require 'ting_yun/agent'
+
+module TingYun
+  module Instrumentation
+    module ThriftHelper
+      def operator result_klass
+        namespaces = result_klass.to_s.split('::')
+        operator_name = namespaces[0].downcase
+        if namespaces.last =~ /_result/
+          operator_name = "#{operator_name}.#{namespaces.last.sub('_result', '').downcase}"
+        elsif namespaces.last =~ /_args/
+          operator_name = "#{operator_name}.#{namespaces.last.sub('_args', '').downcase}"
+        end
+
+        operator_name
+      end
+
+      def operations
+        @operations ||= {}
+      end
+
+      def started_time_and_node(operate)
+        _op_ = operations.delete(operate)
+        time = (_op_ && _op_[:started_time]) or Time.now.to_f
+        node = _op_ && _op_[:node]
+        [time, node]
+      end
+
+
+      def tingyun_socket
+        @iprot.instance_variable_get("@trans").instance_variable_get("@transport")
+      end
+
+      def tingyun_host
+        @tingyun_host ||= tingyun_socket.instance_variable_get("@host") rescue nil
+      end
+
+      def tingyun_port
+        @tingyun_port  ||= tingyun_socket.instance_variable_get("@port") rescue nil
+      end
+
+      def metrics operate
+        state = TingYun::Agent::TransactionState.tl_get
+        metrics = if tingyun_host.nil?
+                    ["External/thrift:%2F%2F#{operate}/#{operate}"]
+                  else
+                    ["External/thrift:%2F%2F#{tingyun_host}:#{tingyun_port}%2F#{operate}/#{operate}"]
+                  end
+        metrics << "External/NULL/ALL"
+
+        if TingYun::Agent::Transaction.recording_web_transaction?
+          metrics << "External/NULL/AllWeb"
+        else
+          metrics << "External/NULL/AllBackground"
+        end
+
+
+        my_data = state.thrift_return_data
+
+
+        if my_data
+          uri = "thrift:%2F%2F#{tingyun_host}:#{tingyun_port}%2F#{operate}/#{operate}"
+          metrics << "cross_app;#{my_data["id"]};#{my_data["action"]};#{uri}"
+        end
+        return metrics
+
+      end
+    end
+  end
+end
+
+
+
+
+TingYun::Support::LibraryDetection.defer do
+  named :thrift
+
+  depends_on do
+    defined?(::Thrift) && defined?(::Thrift::Client) && defined?(::Thrift::BaseProtocol)
+  end
+
+
+  executes do
+    TingYun::Agent.logger.info 'Installing Thrift Instrumentation'
+    require 'ting_yun/support/serialize/json_wrapper'
+  end
+
+  executes do
+    # ::Thrift::Processor.module_eval do
+    #
+    #
+    #
+    #   def same_account?(state)
+    #     server_info = TingYun::Agent.config[:tingyunIdSecret].split('|')
+    #     client_info = (state.client_tingyun_id_secret || '').split('|')
+    #     if !server_info[0].nil? && server_info[0] == client_info[0] && !server_info[0].empty?
+    #       return true
+    #     else
+    #       return false
+    #     end
+    #   end
+    #   def write_result_with_tingyun(result, oprot, name, seqid)
+    #
+    #     state = TingYun::Agent::TransactionState.tl_get
+    #     oprot.write_message_begin(name, ::Thrift::MessageTypes::REPLY, seqid)
+    #
+    #     if state.execution_traced? && same_account?(state)
+    #       class_name = "WebAction/thrift/#{self.class.to_s.split('::').first.downcase}.#{name}"
+    #       state.current_transaction.default_name = class_name
+    #       data = TingYun::Support::Serialize::JSONWrapper.dump("TingyunTxData" => build_payload(state))
+    #       oprot.write_field_begin("TingyunField", 11, 6)
+    #       oprot.write_string(data)
+    #       oprot.write_field_end
+    #       write_result_without_tingyun(result, oprot, name, seqid)
+    #       state.current_transaction.add_agent_attribute(:httpStatus, 200)
+    #       TingYun::Agent::Transaction.stop(state)
+    #     else
+    #       write_result_without_tingyun(result, oprot, name, seqid)
+    #     end
+    #   end
+    #
+    #   def write_error_with_tingyun(err, oprot, name, seqid)
+    #     p 'write_error'
+    #     state = TingYun::Agent::TransactionState.tl_get
+    #     oprot.write_message_begin(name, ::Thrift::MessageTypes::EXCEPTION, seqid)
+    #
+    #     if state.execution_traced? && same_account?(state)
+    #
+    #       class_name = "WebAction/thrift/#{self.class.to_s.split('::').first.downcase}.#{name}"
+    #       state.current_transaction.default_name = class_name
+    #       data = TingYun::Support::Serialize::JSONWrapper.dump("TingyunTxData" => build_payload(state))
+    #       oprot.write_field_begin("TingyunField", 11, 6)
+    #       oprot.write_string(data)
+    #       oprot.write_field_end
+    #       write_result_without_tingyun(err, oprot, name, seqid)
+    #       p 'write_error end'
+    #       state.current_transaction.add_agent_attribute(:httpStatus, 500)
+    #
+    #       TingYun::Agent::Transaction.stop(state)
+    #     else
+    #       write_result_without_tingyun(err, oprot, name, seqid)
+    #     end
+    #   end
+    #
+    #
+    #   def build_payload(state)
+    #     state.web_duration = TingYun::Helper.time_to_millis(Time.now - state.current_transaction.start_time)
+    #     payload = {
+    #         :id => TingYun::Agent.config[:tingyunIdSecret].split('|')[1],
+    #         :action => state.current_transaction.best_name,
+    #         :trId => state.transaction_sample_builder.trace.guid,
+    #         :time => {
+    #             :duration => state.web_duration,
+    #             :qu => state.queue_duration,
+    #             :db => state.sql_duration,
+    #             :ex => state.external_duration,
+    #             :rds => state.rds_duration,
+    #             :mc => state.mc_duration,
+    #             :mon => state.mon_duration,
+    #             :code => execute_duration(state)
+    #         }
+    #     }
+    #     payload[:tr] = 1 if slow_action_tracer?(state)
+    #     payload[:r] = state.client_req_id unless state.client_req_id.nil?
+    #     payload
+    #   end
+    #
+    #   def slow_action_tracer?(state)
+    #     if state.web_duration > TingYun::Agent.config[:'nbs.action_tracer.action_threshold']
+    #       return true
+    #     else
+    #       return false
+    #     end
+    #   end
+    #
+    #   def write_result_without_tingyun(result, oprot, name, seqid)
+    #     result.write(oprot)
+    #     oprot.write_message_end
+    #     oprot.trans.flush
+    #   end
+    #
+    #   def execute_duration(state)
+    #     state.web_duration - state.queue_duration - state.sql_duration - state.external_duration - state.rds_duration - state.mc_duration - state.mon_duration
+    #   end
+    #
+    #   alias :write_result  :write_result_with_tingyun
+    #   alias :write_error  :write_error_with_tingyun
+    #   # alias :write_result_without_tingyun  :write_result
+    # end
+
+    ::Thrift::BaseProtocol.class_eval do
+
+      def skip_with_tingyun(type)
+
+        data = skip_without_tingyun(type)
+        state = TingYun::Agent::TransactionState.tl_get
+        if data.is_a? ::String
+          if data.include?("TingyunTxData")
+
+            my_data = TingYun::Support::Serialize::JSONWrapper.load data.gsub("'",'"')
+
+            state.thrift_return_data = my_data["TingyunTxData"]
+
+            transaction_sampler = ::TingYun::Agent.instance.transaction_sampler
+            transaction_sampler.tl_builder.current_node[:txId] = state.request_guid
+            transaction_sampler.tl_builder.current_node[:txData] = my_data["TingyunTxData"]
+          # elsif data.include?("TingyunID")
+          #   TingYun::Agent::Transaction.start(state, :thrift, :apdex_start_time => Time.now)
+          #   my_data = TingYun::Support::Serialize::JSONWrapper.load data.gsub("'",'"')
+          #   save_referring_transaction_info(state, my_data)
+          end
+        end
+      end
+
+      def save_referring_transaction_info(state,data)
+
+        info = data["TingyunID"].split(';')
+        tingyun_id_secret = info[0]
+        client_transaction_id = info.find do |e|
+          e.match(/x=/)
+        end.split('=')[1] rescue nil
+        client_req_id = info.find do |e|
+          e.match(/r=/)
+        end.split('=')[1] rescue nil
+
+        state.client_tingyun_id_secret = tingyun_id_secret
+        state.client_transaction_id = client_transaction_id
+        state.client_req_id = client_req_id
+      end
+
+      alias :skip_without_tingyun :skip
+      alias :skip  :skip_with_tingyun
+    end
+
+    ::Thrift::Client.module_eval do
+
+      include TingYun::Instrumentation::ThriftHelper
+
+      def send_message_args_with_tingyun(args_class, args = {})
+        state = TingYun::Agent::TransactionState.tl_get
+        return  unless state.execution_traced?
+        cross_app_id  = TingYun::Agent.config[:tingyunIdSecret] or
+            raise TingYun::Agent::CrossAppTracing::Error, "no tingyunIdSecret configured"
+        txn_guid = state.request_guid
+        tingyun_id = "#{cross_app_id};c=1;x=#{txn_guid}"
+
+        data = TingYun::Support::Serialize::JSONWrapper.dump("TingyunID" => tingyun_id)
+        @oprot.write_field_begin("TingyunField", 11, 6)
+        @oprot.write_string(data)
+        @oprot.write_field_end
+        send_message_args_without_tingyun(args_class, args)
+      end
+
+      alias :send_message_args_without_tingyun :send_message_args
+      alias :send_message_args  :send_message_args_with_tingyun
+
+      def send_message_with_tingyun(name, args_class, args = {})
+
+        tag = "#{args_class.to_s.split('::').first.downcase}.#{name}"
+        t0 = Time.now.to_f
+        operations[tag] = {:started_time => t0}
+        state = TingYun::Agent::TransactionState.tl_get
+        return  unless state.execution_traced?
+        stack = state.traced_method_stack
+        node = stack.push_frame(state,:thrift,t0)
+        operations[tag][:node] = node
+
+        send_message_without_tingyun(name, args_class, args)
+      end
+
+      alias :send_message_without_tingyun :send_message
+      alias :send_message  :send_message_with_tingyun
+
+      def send_oneway_message_with_tingyun(name, args_class, args = {})
+        tag = "#{args_class.to_s.split('::').first.downcase}.#{name}"
+        op_started = Time.now.to_f
+        base, *other_metrics = metrics(tag)
+        result = send_oneway_message_without_tingyun(name, args_class, args)
+        duration = (Time.now.to_f - op_started)*1000
+        TingYun::Agent.instance.stats_engine.tl_record_scoped_and_unscoped_metrics(base, other_metrics, duration)
+        result
+      end
+      alias :send_oneway_message_without_tingyun :send_oneway_message
+      alias :send_oneway_message :send_oneway_message_with_tingyun
+
+      def receive_message_with_tingyun(result_klass)
+        state = TingYun::Agent::TransactionState.tl_get
+
+        operate = operator(result_klass)
+
+        t0, node =  started_time_and_node(operate)
+
+
+        result = receive_message_without_tingyun(result_klass)
+        if result.nil? || result.success.nil?
+          e = ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, "#{operate} failed: unknown result")
+          e.instance_variable_set(:@tingyun_klass, metrics(operate)[0])
+          e.instance_variable_set(:@tingyun_external, true)
+          e.instance_variable_set(:@tingyun_code, 1000)
+          e.instance_variable_set(:@tingyun_trace, caller.reject! { |t| t.include?('tingyun_rpm') })
+          TingYun::Agent.notice_error(e)
+        end
+
+        t1 = Time.now.to_f
+        node_name, *other_metrics = metrics(operate)
+        duration = TingYun::Helper.time_to_millis(t1 - t0)
+
+        TingYun::Agent.instance.stats_engine.tl_record_scoped_and_unscoped_metrics(
+            other_metrics.pop, other_metrics, duration
+        )
+        if node
+          node.name = node_name
+          transaction_sampler = ::TingYun::Agent.instance.transaction_sampler
+          transaction_sampler.add_node_info(:uri => "thrift:#{tingyun_host}:#{tingyun_port}/#{operate}")
+          stack = state.traced_method_stack
+          stack.pop_frame(state, node, node_name, t1)
+        end
+
+        result
+      end
+
+      alias :receive_message_without_tingyun :receive_message
+      alias :receive_message :receive_message_with_tingyun
+    end
+  end
+
+end

data/lib/ting_yun/logger/agent_logger.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+# This file is distributed under Ting Yun's license terms.
+
+require 'thread'
+require 'logger'
+require 'ting_yun/logger/log_once'
+require 'ting_yun/logger/memory_logger'
+require 'ting_yun/support/hostname'
+require 'ting_yun/logger/null_logger'
+
+
+module TingYun
+  module Logger
+    class AgentLogger
+      include LogOnce
+
+      def initialize(root = "", override_logger=nil)
+        @already_logged_lock = Mutex.new
+        clear_already_logged
+        create_log(root, override_logger)
+        set_log_level!
+        set_log_format!
+
+        gather_startup_logs
+      end
+
+      def fatal(*msgs, &blk)
+        format_and_send(:fatal, msgs, &blk)
+      end
+
+      def error(*msgs, &blk)
+        format_and_send(:error, msgs, &blk)
+      end
+
+      def warn(*msgs, &blk)
+        format_and_send(:warn, msgs, &blk)
+      end
+
+      def info(*msgs, &blk)
+        format_and_send(:info, msgs, &blk)
+      end
+
+      def debug(*msgs, &blk)
+        format_and_send(:debug, msgs, &blk)
+      end
+
+      def is_startup_logger?
+        @log.is_a?(NullLogger)
+      end
+
+      # Use this when you want to log an exception with explicit control over
+      # the log level that the backtrace is logged at. If you just want the
+      # default behavior of backtraces logged at debug, use one of the methods
+      # above and pass an Exception as one of the args.
+      def log_exception(level, e, backtrace_level=level)
+        @log.send(level, "%p: %s" % [e.class, e.message])
+        @log.send(backtrace_level) do
+          backtrace = backtrace_from_exception(e)
+          if backtrace
+            "Debugging backtrace:\n" + backtrace.join("\n  ")
+          else
+            "No backtrace available."
+          end
+        end
+      end
+
+      def log_formatter=(formatter)
+        @log.formatter = formatter
+      end
+
+      private
+
+      def backtrace_from_exception(e)
+        # We've seen that often the backtrace on a SystemStackError is bunk
+        # so massage the caller instead at a known depth.
+        #
+        # Tests keep us honest about minmum method depth our log calls add.
+        return caller.drop(5) if e.is_a?(SystemStackError)
+
+        e.backtrace
+      end
+
+      # Allows for passing exception.rb in explicitly, which format with backtrace
+      def format_and_send(level, *msgs, &block)
+        check_log_file
+        if block
+          if @log.send("#{level}?")
+            msgs = Array(block.call)
+          else
+            msgs = []
+          end
+        end
+
+        msgs.flatten.each do |item|
+          case item
+            when Exception then
+              log_exception(level, item, :debug)
+            else
+              @log.send(level, item)
+          end
+        end
+        nil
+      end
+
+      def check_log_file
+        unless File.exist? @file_path
+          begin
+            @log = ::Logger.new(@file_path)
+            set_log_format!
+          rescue => e
+            @log = ::Logger.new(STDOUT)
+            warn("check_log_file:  Failed creating logger for file #{file_path}, using standard out for logging.", e)
+          end
+        end
+      end
+
+      def create_log(root, override_logger)
+        if !override_logger.nil?
+          @log = override_logger
+        elsif ::TingYun::Agent.config[:'nbs.agent_enabled'] == false
+          create_null_logger
+        else
+          if wants_stdout?
+            @log = ::Logger.new(STDOUT)
+          else
+            create_log_to_file(root)
+          end
+        end
+      end
+
+      def create_log_to_file(root)
+        path = find_or_create_file_path(::TingYun::Agent.config[:agent_log_file_path], root)
+        if path.nil?
+          @log = ::Logger.new(STDOUT)
+          warn("Error creating log directory #{::TingYun::Agent.config[:agent_log_file_path]}, using standard out for logging.")
+        else
+          @file_path = "#{path}/#{::TingYun::Agent.config[:agent_log_file_name]}"
+          begin
+            @log = ::Logger.new(@file_path)
+          rescue => e
+            @log = ::Logger.new(STDOUT)
+            warn("Failed creating logger for file #{file_path}, using standard out for logging.", e)
+          end
+        end
+      end
+
+      def create_null_logger
+        @log = ::TingYun::Logger::NullLogger.new
+      end
+
+      def wants_stdout?
+        ::TingYun::Agent.config[:agent_log_file_name].upcase == "STDOUT"
+      end
+
+      def find_or_create_file_path(path_setting, root)
+        for abs_path in [File.expand_path(path_setting),
+                         File.expand_path(File.join(root, path_setting))] do
+          if File.directory?(abs_path) || (Dir.mkdir(abs_path) rescue nil)
+            return abs_path[%r{^(.*?)/?$}]
+          end
+        end
+        nil
+      end
+
+      def set_log_level!
+        @log.level = AgentLogger.log_level_for(::TingYun::Agent.config[:agent_log_level])
+      end
+
+      LOG_LEVELS = {
+          "debug" => ::Logger::DEBUG,
+          "info" => ::Logger::INFO,
+          "warn" => ::Logger::WARN,
+          "error" => ::Logger::ERROR,
+          "fatal" => ::Logger::FATAL,
+      }
+
+      def self.log_level_for(level)
+        LOG_LEVELS.fetch(level.to_s.downcase, ::Logger::INFO)
+      end
+
+      def set_log_format!
+        @hostname = TingYun::Support::Hostname.get
+        @prefix = wants_stdout? ? '** [TingYun]' : ''
+        @log.formatter = Proc.new do |severity, timestamp, progname, msg|
+          "#{@prefix}[#{timestamp.strftime("%m/%d/%y %H:%M:%S %z")} #{@hostname} (#{$$})] #{severity} : #{msg}\n"
+        end
+      end
+
+      #send the statup log info from memory to the agent log
+      def gather_startup_logs
+        StartupLogger.instance.dump(self)
+      end
+
+    end
+  end
+end

data/lib/ting_yun/logger/log_once.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+# This file is distributed under Ting Yun's license terms.
+
+module TingYun
+  module Logger
+    module LogOnce
+      NUM_LOG_ONCE_KEYS = 1000
+
+      def log_once(level, key, *msgs)
+        @already_logged_lock.synchronize do
+          return if @already_logged.include?(key)
+
+          if @already_logged.size >= NUM_LOG_ONCE_KEYS && key.kind_of?(String)
+            # The reason for preventing too many keys in `logged` is for
+            # memory concerns.
+            # The reason for checking the type of the key is that we always want
+            # to allow symbols to log, since there are very few of them.
+            # The assumption here is that you would NEVER pass dynamically-created
+            # symbols, because you would never create symbols dynamically in the
+            # first place, as that would already be a memory leak in most Rubies,
+            # even if we didn't hang on to them all here.
+            return
+          end
+
+          @already_logged[key] = true
+        end
+
+        self.send(level, *msgs)
+      end
+
+      def clear_already_logged
+        @already_logged_lock.synchronize do
+          @already_logged = {}
+        end
+      end
+    end
+  end
+end