tingyun_rpm 1.0.6

data/lib/ting_yun/agent/database.rb

@@ -0,0 +1,410 @@
+# encoding: utf-8
+
+require 'ting_yun/support/helper'
+module TingYun
+  module Agent
+    module Database
+
+      MAX_QUERY_LENGTH = 16384
+
+      extend self
+
+
+      def obfuscate_sql(sql)
+        Obfuscator.instance.obfuscator.call(sql)
+      end
+
+
+      def capture_query(query)
+        TingYun::Helper.correctly_encoded(truncate_query(query))
+      end
+
+      def truncate_query(query)
+        if query.length > (MAX_QUERY_LENGTH - 4)
+          query[0..MAX_QUERY_LENGTH - 4] + '...'
+        else
+          query
+        end
+      end
+
+
+      RECORD_FOR = [:raw, :obfuscated].freeze
+
+      def should_record_sql?(key)
+        RECORD_FOR.include?(record_sql_method(key.to_sym))
+      end
+
+      def record_sql_method(key)
+
+        case Agent.config[key].to_s
+          when 'off'
+            :off
+          when 'raw'
+            :raw
+          else
+            :obfuscated
+        end
+      end
+
+      def should_action_collect_explain_plans?
+        should_record_sql?("nbs.action_tracer.record_sql") &&
+            Agent.config["nbs.action_tracer.explain_enabled".to_sym]
+      end
+
+      def explain_sql(sql, config, explainer=nil)
+        return nil unless sql && explainer && config
+        _sql = sql.split(";\n")[0] # only explain the first
+        explain_plan = explain(_sql, config, explainer)
+        return explain_plan || {"dialect"=> nil, "keys"=>[], "values"=>[]}
+      end
+
+      SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
+
+      def explain(sql, config, explainer=nil)
+
+        return unless explainer && is_select?(sql)
+
+        if sql[-3,3] == '...'
+          TingYun::Agent.logger.debug('Unable to collect explain plan for truncated query.')
+          return
+        end
+
+        if parameterized?(sql)
+          TingYun::Agent.logger.debug('Unable to collect explain plan for parameterized query.')
+          return
+        end
+
+        adapter = adapter_from_config(config)
+        if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
+          TingYun::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
+          return
+        end
+
+        handle_exception_in_explain do
+          plan = explainer.call(config, sql)
+          return process_resultset(plan, adapter) if plan
+        end
+      end
+
+      def adapter_from_config(config)
+        if config[:adapter]
+          return config[:adapter].to_s
+        elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
+          # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
+          # jdbc-sqlite3 gems.
+          return $1
+        end
+      end
+
+
+      def parameterized?(sql)
+        Obfuscator.instance.obfuscate_single_quote_literals(sql) =~ /\$\d+/
+      end
+
+      def is_select?(sql)
+        parse_operation_from_query(sql) == 'select'
+      end
+
+      def process_resultset(results ,adapter)
+        case adapter.to_s
+          when 'postgres', 'postgresql'
+            process_explain_results_postgres(results)
+          when 'mysql2'
+            process_explain_results_mysql2(results)
+          when 'mysql'
+            process_explain_results_mysql(results)
+          when 'sqlite'
+            process_explain_results_sqlite(results)
+        end
+      end
+
+      QUERY_PLAN = 'QUERY PLAN'.freeze
+
+      def process_explain_results_postgres(results)
+        if results.is_a?(String)
+          query_plan_string = results
+        else
+          lines = []
+          results.each { |row| lines << row[QUERY_PLAN] }
+          query_plan_string = lines.join("\n")
+        end
+
+        unless record_sql_method == :raw
+          query_plan_string = Obfuscator.instance.obfuscate_postgres_explain(query_plan_string)
+        end
+        values = query_plan_string.split("\n").map { |line| [line] }
+
+        {"dialect"=> "PostgreSQL", "keys"=>[QUERY_PLAN], "values"=>values}
+      end
+
+      def string_explain_plan_results(adpater, results)
+        {"dialect"=> adpater, "keys"=>[], "values"=>[results]}
+      end
+
+      def process_explain_results_mysql2(results)
+        return string_explain_plan_results("MySQL", results) if results.is_a?(String)
+        headers = results.fields
+        values  = []
+        results.each { |row| values << row }
+        {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
+      end
+
+      def process_explain_results_mysql(results)
+        return string_explain_plan_results("MySQL", results) if results.is_a?(String)
+        headers = []
+        values  = []
+        if results.is_a?(Array)
+          # We're probably using the jdbc-mysql gem for JRuby, which will give
+          # us an array of hashes.
+          headers = results.first.keys
+          results.each do |row|
+            values << headers.map { |h| row[h] }
+          end
+        else
+          # We're probably using the native mysql driver gem, which will give us
+          # a Mysql::Result object that responds to each_hash
+          results.each_hash do |row|
+            headers = row.keys
+            values << headers.map { |h| row[h] }
+          end
+        end
+        {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
+      end
+
+      SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
+
+      def process_explain_results_sqlite(results)
+        return string_explain_plan_results("sqlite", results) if results.is_a?(String)
+        headers = SQLITE_EXPLAIN_COLUMNS
+        values  = []
+        results.each do |row|
+          values << headers.map { |h| row[h] }
+        end
+        {"dialect"=> "sqlite", "keys"=>headers, "values"=>values}
+      end
+
+
+
+      KNOWN_OPERATIONS = [
+          'alter',
+          'select',
+          'update',
+          'delete',
+          'insert',
+          'create',
+          'show',
+          'set',
+          'exec',
+          'execute',
+          'call'
+      ]
+
+      SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
+      EMPTY_STRING      = ''.freeze
+
+      def parse_operation_from_query(sql)
+        sql = TingYun::Helper.correctly_encoded(sql).gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
+        if sql =~ /(\w+)/
+          op = $1.downcase
+          return op if KNOWN_OPERATIONS.include?(op)
+        end
+      end
+
+
+      def handle_exception_in_explain
+        yield
+      rescue => e
+        ::TingYun::Agent.logger.error("Error getting query plan:", e)
+        nil
+      end
+
+
+      def get_connection(config, &connector)
+        ConnectionManager.instance.get_connection(config, &connector)
+      end
+
+      def close_connections
+        ConnectionManager.instance.close_connections
+      end
+
+      # Returns a cached connection for a given ActiveRecord
+      # configuration - these are stored or reopened as needed, and if
+      # we cannot get one, we ignore it and move on without explaining
+      # the sql
+      class ConnectionManager
+        include Singleton
+
+        def get_connection(config, &connector)
+          @connections ||= {}
+
+          connection = @connections[config]
+
+          return connection if connection
+
+          begin
+            @connections[config] = connector.call(config)
+          rescue => e
+            ::TingYun::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
+            nil
+          end
+        end
+
+        # Closes all the connections in the internal connection cache
+        def close_connections
+          @connections ||= {}
+          @connections.values.each do |connection|
+            begin
+              connection.disconnect!
+            rescue
+            end
+          end
+
+          @connections = {}
+        end
+      end
+
+      class Statement
+        attr_accessor :sql, :config, :explainer
+
+        def initialize(sql, config={}, explainer=nil)
+          @sql = TingYun::Agent::Database.capture_query(sql)
+          @config = config
+          @explainer = explainer
+        end
+
+        def adapter
+          config && config[:adapter]
+        end
+      end
+
+      #混淆器
+      class Obfuscator
+        include Singleton
+
+        attr_reader :obfuscator
+
+        def initialize
+          reset
+        end
+
+        def reset
+          @obfuscator = method(:default_sql_obfuscator)
+        end
+
+        QUERY_TOO_LARGE_MESSAGE     = "Query too large (over 16k characters) to safely obfuscate"
+        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
+
+        def default_sql_obfuscator(sql)
+          stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
+
+          if stmt.sql[-3,3] == '...'
+            return QUERY_TOO_LARGE_MESSAGE
+          end
+
+          obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
+
+          obfuscated = obfuscate_numeric_literals(stmt.sql)
+
+          if obfuscate_double_quotes
+            obfuscated = obfuscate_quoted_literals(obfuscated)
+            obfuscated = remove_comments(obfuscated)
+            if contains_quotes?(obfuscated)
+              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
+            end
+          else
+            obfuscated = obfuscate_single_quote_literals(obfuscated)
+            obfuscated = remove_comments(obfuscated)
+            if contains_single_quotes?(obfuscated)
+              obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
+            end
+          end
+
+
+          obfuscated.to_s # return back to a regular String
+        end
+
+        QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
+        LABEL_LINE_REGEX     = /^([^:\n]*:\s+).*$/.freeze
+
+        def obfuscate_postgres_explain(explain)
+          explain.gsub!(QUOTED_STRINGS_REGEX) do |match|
+            match.start_with?('"') ? match : '?'
+          end
+          explain.gsub!(LABEL_LINE_REGEX,   '\1?')
+          explain
+        end
+
+        module ObfuscationHelpers
+          # Note that the following two regexes are applied to a reversed version
+          # of the query. This is why the backslash escape sequences (\' and \")
+          # appear reversed within them.
+          #
+          # Note that some database adapters (notably, PostgreSQL with
+          # standard_conforming_strings on and MySQL with NO_BACKSLASH_ESCAPES on)
+          # do not apply special treatment to backslashes within quoted string
+          # literals. We don't have an easy way of determining whether the
+          # database connection from which a query was captured was operating in
+          # one of these modes, but the obfuscation is done in such a way that it
+          # should not matter.
+          #
+          # Reversing the query string before obfuscation allows us to get around
+          # the fact that a \' appearing within a string may or may not terminate
+          # the string, because we know that a string cannot *start* with a \'.
+          REVERSE_SINGLE_QUOTES_REGEX = /'(?:''|'\\|[^'])*'/
+          REVERSE_ANY_QUOTES_REGEX    = /'(?:''|'\\|[^'])*'|"(?:""|"\\|[^"])*"/
+
+          NUMERICS_REGEX = /\b\d+\b/
+
+          # We take a conservative, overly-aggressive approach to obfuscating
+          # comments, and drop everything from the query after encountering any
+          # character sequence that could be a comment initiator. We do this after
+          # removal of string literals to avoid accidentally over-obfuscating when
+          # a string literal contains a comment initiator.
+          SQL_COMMENT_REGEX = Regexp.new('(?:/\*|--|#).*', Regexp::MULTILINE).freeze
+
+          # We use these to check whether the query contains any quote characters
+          # after obfuscation. If so, that's a good indication that the original
+          # query was malformed, and so our obfuscation can't reliabily find
+          # literals. In such a case, we'll replace the entire query with a
+          # placeholder.
+          LITERAL_SINGLE_QUOTE = "'".freeze
+          LITERAL_DOUBLE_QUOTE = '"'.freeze
+
+          PLACEHOLDER = '?'.freeze
+
+          def obfuscate_single_quote_literals(sql)
+            obfuscated = sql.reverse
+            obfuscated.gsub!(REVERSE_SINGLE_QUOTES_REGEX, PLACEHOLDER)
+            obfuscated.reverse!
+            obfuscated
+          end
+
+          def obfuscate_quoted_literals(sql)
+            obfuscated = sql.reverse
+            obfuscated.gsub!(REVERSE_ANY_QUOTES_REGEX, PLACEHOLDER)
+            obfuscated.reverse!
+            obfuscated
+          end
+
+          def obfuscate_numeric_literals(sql)
+            sql.gsub(NUMERICS_REGEX, PLACEHOLDER)
+          end
+
+          def remove_comments(sql)
+            sql.gsub(SQL_COMMENT_REGEX, PLACEHOLDER)
+          end
+
+          def contains_single_quotes?(str)
+            str.include?(LITERAL_SINGLE_QUOTE)
+          end
+
+          def contains_quotes?(str)
+            str.include?(LITERAL_SINGLE_QUOTE) || str.include?(LITERAL_DOUBLE_QUOTE)
+          end
+        end
+        include ObfuscationHelpers
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/datastore/metric_helper.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require 'ting_yun/agent/transaction/transaction_state'
+module TingYun
+  module Agent
+    module Datastore
+      module MetricHelper
+
+
+        ALL_WEB = "AllWeb".freeze
+        ALL_BACKGROUND = "AllBackground".freeze
+        ALL = "All".freeze
+
+        NOSQL = %w(MongoDB Redis Memcached).freeze
+
+        def self.checkNosql(product)
+          NOSQL.include?(product)
+        end
+
+        def self.metric_name(product, collection, operation)
+          if checkNosql(product)
+            if product == 'MongoDB'
+              "#{product}/#{collection}/#{operation}"
+            else
+              "#{product}/NULL/#{operation}"
+            end
+          else
+            "Database #{product}/#{collection}/#{operation}"
+          end
+        end
+
+        def self.product_suffixed_rollup(product,suffix)
+          if checkNosql(product)
+            "#{product}/NULL/#{suffix}"
+          else
+            "Database #{product}/NULL/#{suffix}"
+          end
+        end
+
+
+
+        def self.metrics_for(product, operation, collection = nil, generic_product = nil)
+          operation = operation.to_s.upcase
+          if overrides = overridden_operation_and_collection   # [method, model_name, product]
+            if should_override?(overrides, product, generic_product)
+              operation  = overrides[0] || operation
+              collection = overrides[1] || collection
+            end
+          end
+          metrics = [ALL_WEB,ALL]
+          metrics << operation
+          metrics = metrics.map do |suffix|
+            product_suffixed_rollup(product,suffix)
+          end
+
+          metrics.unshift metric_name(product, collection, operation)
+          metrics
+        end
+
+        # Allow Transaction#with_database_metric_name to override our
+        # collection and operation
+        def self.overridden_operation_and_collection #THREAD_LOCAL_ACCESS
+          state = TingYun::Agent::TransactionState.tl_get
+          txn   = state.current_transaction
+          txn ? txn.instrumentation_state[:datastore_override] : nil
+        end
+
+        # If the override declared a product affiliation, abide by that
+        # ActiveRecord has database-specific product names, so we recognize
+        # it by the generic_product it passes.
+        def self.should_override?(overrides, product, generic_product)
+          override_product = overrides[2]
+
+          override_product.nil? ||
+              override_product == product ||
+              override_product == generic_product
+        end
+
+      end
+    end
+  end
+end
+

data/lib/ting_yun/agent/datastore/mongo.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+require 'ting_yun/support/version_number'
+module TingYun
+  module Agent
+    module Datastore
+      module Mongo
+
+        def self.monitoring_enabled?
+          defined?(::Mongo::Monitoring)
+        end
+
+        def self.version_1_10_or_later?
+          # Again, no VERSION constant in 1.x, so we have to rely on constant checks
+          defined?(::Mongo::CollectionOperationWriter)
+        end
+
+        def self.unsupported_2x?
+          defined?(::Mongo::VERSION) && TingYun::Support::VersionNumber.new(::Mongo::VERSION).major_version == 2
+        end
+
+        def self.supported_version?
+          # No version constant in < 2.0 versions of Mongo :(
+          defined?(::Mongo) && (defined?(::Mongo::MongoClient) || monitoring_enabled?)
+        end
+
+        # def self.transform_operation(operation)
+        #   t_operation = case operation.to_s.upcase
+        #                 when 'DELETE', 'FIND_AND_REMOVE', 'DELETEINDEXS', 'REMOVE'                       then 'destroy'
+        #                 when 'INSERT'                                                                    then 'INSERT'
+        #                 when 'UPDATE', 'RENAMECOLLECTION', 'REINDEX'                                     then 'UPDATE'
+        #                 when 'CREATE', 'FIND_AND_MODIFY', 'CREATEINDEXS', 'CREATEINDEX', 'REPINDEX'      then 'SAVE'
+        #                 when 'QUERY', 'COUNT', 'GET_MORE', 'AGGREGATE', 'FIND', 'FINDONE', 'GROUP'       then 'SECECT'
+        #                 else
+        #                   nil
+        #                 end
+        #   t_operation
+        # end
+        def self.transform_operation(operation)
+          operation.to_s.upcase
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/datastore.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+require 'ting_yun/agent/datastore/metric_helper'
+require 'ting_yun/agent/method_tracer'
+
+
+module TingYun
+  module Agent
+    module Datastore
+
+
+      def self.wrap(product, operation, collection = nil, callback = nil)
+        return yield unless operation
+
+        metrics = TingYun::Agent::Datastore::MetricHelper.metrics_for(product, operation, collection)
+
+        scoped_metric = metrics.last
+
+        TingYun::Agent::MethodTracer.trace_execution_scoped(metrics) do
+          t0 = Time.now
+          begin
+            result = yield
+          ensure
+            elapsed_time = (Time.now - t0).to_f
+            if callback
+              callback.call(result, scoped_metric, elapsed_time)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/dispatcher.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+module TingYun
+  module Agent
+    class Dispatcher
+      attr_accessor :starting_pid
+
+      def initialize(events, after_forker=TingYun::Agent)
+        # We intentionally don't set our pid as started at this point.
+        # Startup routines must call mark_started when they consider us set!
+        @starting_pid = nil
+        @after_forker = after_forker
+
+        if events
+          events.subscribe(:start_transaction, &method(:on_transaction))
+        end
+      end
+
+      def on_transaction(*_)
+        return unless needs_restart?
+
+        restart_harvest_thread
+      end
+
+      def mark_started(pid = Process.pid)
+        @starting_pid = pid
+      end
+
+      def needs_restart?(pid = Process.pid)
+        @starting_pid != pid
+      end
+
+      def restart_harvest_thread
+        @after_forker.after_fork(:force_reconnect => true)
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/event/event_listener.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+module TingYun
+  module Agent
+    module Event
+      class EventListener
+
+        attr_accessor :allocation
+
+        def initialize
+          @events = {}
+          @allocation = 100
+        end
+
+        def subscribe(event, &handler)
+          @events[event] ||= []
+          @events[event] << handler
+          check_allocation(event)
+        end
+
+        def check_allocation(event)
+          if @events[event].size > @allocation
+            TingYun::Agent.logger.debug("Run-away event subscription on #{event}? Subscribed #{count}")
+          end
+        end
+
+        def clear
+          @events.clear
+        end
+
+
+        def notify(event, *args)
+          return unless @events.has_key?(event)
+
+          @events[event].each do |handler|
+            begin
+              handler.call(*args)
+            rescue => err
+              TingYun::Agent.logger.debug("Failure during notify for #{event}", err)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+