tingyun_rpm 1.1.4.2 → 1.2.0

data/lib/ting_yun/agent/database/connection_manager.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+module TingYun
+  module Agent
+    module Database
+      # Returns a cached connection for a given ActiveRecord
+      # configuration - these are stored or reopened as needed, and if
+      # we cannot get one, we ignore it and move on without explaining
+      # the sql
+      class ConnectionManager
+        include Singleton
+
+        def get_connection(config, &connector)
+          @connections ||= {}
+
+          connection = @connections[config]
+
+          return connection if connection
+
+          begin
+            @connections[config] = connector.call(config)
+          rescue => e
+            ::TingYun::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
+            nil
+          end
+        end
+
+        # Closes all the connections in the internal connection cache
+        def close_connections
+          @connections ||= {}
+          @connections.values.each do |connection|
+            begin
+              connection.disconnect!
+            rescue
+            end
+          end
+
+          @connections = {}
+        end
+      end
+
+    end
+  end
+end

data/lib/ting_yun/agent/database/explain_plan_helpers.rb

@@ -0,0 +1,173 @@
+# encoding: utf-8
+require 'ting_yun/agent/database/obfuscator'
+
+module TingYun
+  module Agent
+    module Database
+      module ExplainPlanHelpers
+
+        def handle_exception_in_explain
+          yield
+        rescue => e
+          begin
+            # guarantees no throw from explain_sql
+            ::TingYun::Agent.logger.error("Error getting query plan:", e)
+            nil
+          rescue
+            # double exception. throw up your hands
+            nil
+          end
+        end
+
+        def is_select?(sql)
+          parse_operation_from_query(sql) == 'select'
+        end
+
+        def parameterized?(sql)
+          TingYun::Agent::Database::Obfuscator.instance.obfuscate_single_quote_literals(sql) =~ /\$\d+/
+        end
+
+        SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
+        EMPTY_STRING      = ''.freeze
+
+
+        KNOWN_OPERATIONS = [
+            'alter',
+            'select',
+            'update',
+            'delete',
+            'insert',
+            'create',
+            'show',
+            'set',
+            'exec',
+            'execute',
+            'call'
+        ]
+
+        def parse_operation_from_query(sql)
+          sql = TingYun::Helper.correctly_encoded(sql).gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
+          if sql =~ /(\w+)/
+            op = $1.downcase
+            return op if KNOWN_OPERATIONS.include?(op)
+          end
+        end
+
+
+
+
+        POSTGRES_PREFIX = 'postgres'.freeze
+        MYSQL_PREFIX    = 'mysql'.freeze
+        MYSQL2_PREFIX   = 'mysql2'.freeze
+        SQLITE_PREFIX   = 'sqlite'.freeze
+
+        def symbolized_adapter(adapter)
+          if adapter.start_with? POSTGRES_PREFIX
+            :postgres
+          elsif adapter == MYSQL_PREFIX
+            :mysql
+            # For the purpose of fetching explain plans, we need to maintain the distinction
+            # between usage of mysql and mysql2. Obfuscation is the same, though.
+          elsif adapter == MYSQL2_PREFIX
+            :mysql2
+          elsif adapter.start_with? SQLITE_PREFIX
+            :sqlite
+          else
+            adapter.to_sym
+          end
+        end
+
+
+        def process_resultset(results, adapter)
+          if adapter == :postgres
+            return process_explain_results_postgres(results)
+          elsif defined?(::ActiveRecord::Result) && results.is_a?(::ActiveRecord::Result)
+            # Note if adapter is mysql, will only have headers, not values
+            return [results.columns, results.rows]
+          elsif results.is_a?(String)
+            return string_explain_plan_results(results)
+          end
+
+          case adapter
+            when :mysql2
+              process_explain_results_mysql2(results)
+            when :mysql
+              process_explain_results_mysql(results)
+            when :sqlite
+              process_explain_results_sqlite(results)
+            else
+              return {}
+          end
+        end
+
+        QUERY_PLAN = 'QUERY PLAN'.freeze
+
+        def process_explain_results_postgres(results)
+          if defined?(::ActiveRecord::Result) && results.is_a?(::ActiveRecord::Result)
+            query_plan_string = results.rows.join("\n")
+          elsif results.is_a?(String)
+            query_plan_string = results
+          else
+            lines = []
+            results.each { |row| lines << row[QUERY_PLAN] }
+            query_plan_string = lines.join("\n")
+          end
+
+          unless TingYun::Agent::Database.record_sql_method("nbs.action_tracer.record_sql") == :raw
+            query_plan_string = TingYun::Agent::Database::Obfuscator.instance.obfuscate_postgres_explain(query_plan_string)
+          end
+          values = query_plan_string.split("\n").map { |line| [line] }
+
+          {"dialect"=> "PostgreSQL", "keys"=>[QUERY_PLAN], "values"=>values}
+        end
+
+
+        def string_explain_plan_results(adpater, results)
+          {"dialect"=> adpater, "keys"=>[], "values"=>[results]}
+        end
+
+        def process_explain_results_mysql2(results)
+          headers = results.fields
+          values  = []
+          results.each { |row| values << row }
+          {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
+        end
+
+        def process_explain_results_mysql(results)
+          headers = []
+          values  = []
+          if results.is_a?(Array)
+            # We're probably using the jdbc-mysql gem for JRuby, which will give
+            # us an array of hashes.
+            headers = results.first.keys
+            results.each do |row|
+              values << headers.map { |h| row[h] }
+            end
+          else
+            # We're probably using the native mysql driver gem, which will give us
+            # a Mysql::Result object that responds to each_hash
+            results.each_hash do |row|
+              headers = row.keys
+              values << headers.map { |h| row[h] }
+            end
+          end
+          {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
+        end
+
+        SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
+
+        def process_explain_results_sqlite(results)
+          headers = SQLITE_EXPLAIN_COLUMNS
+          values  = []
+          results.each do |row|
+            values << headers.map { |h| row[h] }
+          end
+          {"dialect"=> "sqlite", "keys"=>headers, "values"=>values}
+        end
+
+
+
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/database/obfuscator.rb

@@ -0,0 +1,151 @@
+# encoding: utf-8
+
+module TingYun
+  module Agent
+    module Database
+
+      #混淆器
+      class Obfuscator
+        include Singleton
+
+        attr_reader :obfuscator
+
+        def initialize
+          reset
+        end
+
+        def reset
+          @obfuscator = method(:default_sql_obfuscator)
+        end
+
+        QUERY_TOO_LARGE_MESSAGE     = "Query too large (over 16k characters) to safely obfuscate"
+        FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
+
+        def default_sql_obfuscator(sql)
+          stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
+
+          if stmt.sql[-3,3] == '...'
+            return QUERY_TOO_LARGE_MESSAGE
+          end
+
+          obfuscate(stmt.sql, stmt.adapter).to_s
+        end
+
+
+
+
+
+
+        module ObfuscationHelpers
+          COMPONENTS_REGEX_MAP = {
+              :single_quotes => /'(?:[^']|'')*?(?:\\'.*|'(?!'))/,
+              :double_quotes => /"(?:[^"]|"")*?(?:\\".*|"(?!"))/,
+              :dollar_quotes => /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/,
+              :uuids => /\{?(?:[0-9a-fA-F]\-*){32}\}?/,
+              :numeric_literals => /\b-?(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/,
+              :boolean_literals => /\b(?:true|false|null)\b/i,
+              :hexadecimal_literals => /0x[0-9a-fA-F]+/,
+              :comments => /(?:#|--).*?(?=\r|\n|$)/i,
+              :multi_line_comments => /\/\*(?:[^\/]|\/[^*])*?(?:\*\/|\/\*.*)/,
+              :oracle_quoted_strings => /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/
+          }
+
+          DIALECT_COMPONENTS = {
+              :fallback   => COMPONENTS_REGEX_MAP.keys,
+              :mysql      => [:single_quotes, :double_quotes, :numeric_literals, :boolean_literals,
+                              :hexadecimal_literals, :comments, :multi_line_comments],
+              :postgres   => [:single_quotes, :dollar_quotes, :uuids, :numeric_literals,
+                              :boolean_literals, :comments, :multi_line_comments],
+              :sqlite     => [:single_quotes, :numeric_literals, :boolean_literals, :hexadecimal_literals,
+                              :comments, :multi_line_comments],
+              :oracle     => [:single_quotes, :oracle_quoted_strings, :numeric_literals, :comments,
+                              :multi_line_comments],
+              :cassandra  => [:single_quotes, :uuids, :numeric_literals, :boolean_literals,
+                              :hexadecimal_literals, :comments, :multi_line_comments]
+          }
+
+          # We use these to check whether the query contains any quote characters
+          # after obfuscation. If so, that's a good indication that the original
+          # query was malformed, and so our obfuscation can't reliably find
+          # literals. In such a case, we'll replace the entire query with a
+          # placeholder.
+          CLEANUP_REGEX = {
+              :mysql => /'|"|\/\*|\*\//,
+              :mysql2 => /'|"|\/\*|\*\//,
+              :postgres => /'|\/\*|\*\/|\$(?!\?)/,
+              :sqlite => /'|\/\*|\*\//,
+              :cassandra => /'|\/\*|\*\//,
+              :oracle => /'|\/\*|\*\//,
+              :oracle_enhanced => /'|\/\*|\*\//
+          }
+
+
+          QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
+          LABEL_LINE_REGEX     = /^([^:\n]*:\s+).*$/.freeze
+
+
+          def obfuscate_postgres_explain(sql)
+            sql.gsub!(QUOTED_STRINGS_REGEX) do |match|
+              match.start_with?('"') ? match : '?'
+            end
+
+            sql.gsub!(LABEL_LINE_REGEX,   '\1?')
+            sql
+          end
+
+
+          PLACEHOLDER = '?'.freeze
+          FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation".freeze
+
+
+
+          def obfuscate_single_quote_literals(sql)
+            return sql unless sql =~ COMPONENTS_REGEX_MAP[:single_quotes]
+            sql.gsub(COMPONENTS_REGEX_MAP[:single_quotes], PLACEHOLDER)
+          end
+
+          def self.generate_regex(dialect)
+            components = DIALECT_COMPONENTS[dialect]
+            Regexp.union(components.map{|component| COMPONENTS_REGEX_MAP[component]})
+          end
+
+          MYSQL_COMPONENTS_REGEX = self.generate_regex(:mysql)
+          POSTGRES_COMPONENTS_REGEX = self.generate_regex(:postgres)
+          SQLITE_COMPONENTS_REGEX = self.generate_regex(:sqlite)
+          ORACLE_COMPONENTS_REGEX = self.generate_regex(:oracle)
+          CASSANDRA_COMPONENTS_REGEX = self.generate_regex(:cassandra)
+          FALLBACK_REGEX = self.generate_regex(:fallback)
+
+          def obfuscate(sql, adapter)
+            case adapter
+              when :mysql, :mysql2
+                regex = MYSQL_COMPONENTS_REGEX
+              when :postgres
+                regex = POSTGRES_COMPONENTS_REGEX
+              when :sqlite
+                regex = SQLITE_COMPONENTS_REGEX
+              when :oracle, :oracle_enhanced
+                regex = ORACLE_COMPONENTS_REGEX
+              when :cassandra
+                regex = CASSANDRA_COMPONENTS_REGEX
+              else
+                regex = FALLBACK_REGEX
+            end
+            obfuscated = sql.gsub(regex, PLACEHOLDER)
+            obfuscated = FAILED_TO_OBFUSCATE_MESSAGE if detect_unmatched_pairs(obfuscated, adapter)
+            obfuscated
+          end
+
+          def detect_unmatched_pairs(obfuscated, adapter)
+            if CLEANUP_REGEX[adapter]
+              CLEANUP_REGEX[adapter].match(obfuscated)
+            else
+              CLEANUP_REGEX[:mysql].match(obfuscated)
+            end
+          end
+        end
+        include ObfuscationHelpers
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/database/statement.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+require 'ting_yun/support/helper'
+require 'ting_yun/agent/database/explain_plan_helpers'
+
+module TingYun
+  module Agent
+    module Database
+      class Statement
+        include TingYun::Agent::Database::ExplainPlanHelpers
+
+        attr_accessor :sql, :config, :explainer, :binds, :name
+
+        def initialize(sql, config={}, explainer=nil, binds=[], name=DEFAULT_QUERY_NAME)
+          @sql = TingYun::Agent::Database.capture_query(sql)
+          @config = config
+          @explainer = explainer
+          @binds = binds
+          @name = name
+        end
+
+        def adapter
+          return unless @config
+
+          @adapter ||= if @config[:adapter]
+                         symbolized_adapter(@config[:adapter].to_s.downcase)
+                       elsif @config[:uri] && @config[:uri].to_s =~ /^jdbc:([^:]+):/
+                         # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or jdbc-sqlite3 gems.
+                         symbolized_adapter($1)
+                       else
+                         nil
+                       end
+        end
+
+
+
+        SUPPORTED_ADAPTERS_FOR_EXPLAIN = [:postgres, :mysql2, :mysql, :sqlite]
+
+        def explain
+          return unless explainable?
+          handle_exception_in_explain do
+            plan = explainer.call(self)
+            return process_resultset(plan, adapter) if plan
+          end
+        end
+
+
+        def explainable?
+          return false unless @explainer && is_select?(sql)
+
+          if sql[-3,3] == '...'
+            TingYun::Agent.logger.debug('Unable to collect explain plan for truncated query.')
+            return false
+          end
+
+          if parameterized?(@sql) && @binds.empty?
+            TingYun::Agent.logger.debug('Unable to collect explain plan for parameter-less parameterized query.')
+            return false
+          end
+
+          if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
+            TingYun::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
+            return false
+          end
+
+          true
+        end
+      end
+    end
+  end
+end

data/lib/ting_yun/agent/event/event_loop.rb

@@ -146,10 +146,9 @@ module TingYun
           @subscriptions[event].each do |s|
             begin
               s.call(*args)
-            rescue TingYun::Support::Exception::ExpiredConfigurationException, TingYun::Support::Exception::InvalidDataTokenException, TingYun::Support::Exception::InvalidDataException
-              raise
             rescue => e
               errors << e
+              raise
             end
           end