rdavila-query_reviewer 0.1.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,51 @@
+QueryReviewer
+=============
+
+QueryReviewer is an advanced SQL query analyzer.  It accomplishes the following goals:
+
+ * View all EXPLAIN output for all SELECT queries to generate a page (and optionally SHOW PROFILE ALL)
+ * Rate a page's SQL usage into one of three categories: OK, WARNING, CRITICAL
+ * Attach meaningful warnings to individual queries, and collections of queries
+ * Display interactive summary on page
+
+All you have to do is install it.  You can optionally run:
+  rake query_reviewer:setup
+
+Which will create config/query_reviewer.yml, see below for what these options mean.  If you don't have a config file,
+the plugin will use the default in vendor/plugins/query_reviewer.
+
+
+Configuration
+=============
+
+The configuration file allows you to set configuration parameters shared across all rails environment, as well as
+overriding those shared parameteres with environment-specific parameters (such as disabling analysis on production!)
+
+ * enabled: whether any output or query analysis is performed.  Set this false in production!
+ * inject_view: controls whether the output automatically is injected before the </body> in HTML output.
+ * profiling: when enabled, runs the MySQL SET PROFILING=1 for queries longer than the warn_duration_threshold / 2.0
+ * production_data: whether the duration of a query should be taken into account (if you don't have real data, don't let query duration effect you!)
+
+ * stack_trace_lines: number of lines of call stack to include in the "short" version of the stack trace
+ * trace_includes_vendor: whether the "short" verison of the stack trace should include files in /vendor
+ * trace_includes_lib: whether the "short" verison of the stack trace should include files in /lib
+
+ * warn_severity: the severity of problem that merits "WARNING" status
+ * critical_severity: the severity of problem that merits "CRITICAL" status
+
+ * warn_query_count: the number of queries in a single request that merits "WARNING" status
+ * critical_query_count: the number of queries in a single request that merits "CRITICAL" status
+
+ * warn_duration_threshold: how long a query must take in seconds (float) before it's considered "WARNING"
+ * critical_duration_threshold: how long a query must take in seconds (float) before it's considered "CRITICIAL"
+
+
+Example
+=======
+
+If you disable the inject_view option, you'll need to manually put the analyzer's output into your view:
+
+<%= query_review_output %>
+
+
+Copyright (c) 2007-2008 Kongregate & David Stevenson, released under the MIT license

data/Rakefile

@@ -0,0 +1,22 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the query_reviewer plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the query_reviewer plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'QueryReviewer'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/init.rb

@@ -0,0 +1,13 @@
+# Include hook code here
+
+require 'query_reviewer'
+
+if QueryReviewer.enabled?
+  ActiveRecord::ConnectionAdapters::MysqlAdapter.send(:include, QueryReviewer::MysqlAdapterExtensions)
+  ActionController::Base.send(:include, QueryReviewer::ControllerExtensions)
+  Array.send(:include, QueryReviewer::ArrayExtensions)
+  
+  if ActionController::Base.respond_to?(:append_view_path)
+    ActionController::Base.append_view_path(File.dirname(__FILE__) + "/lib/query_reviewer/views")
+  end
+end

data/lib/query_reviewer.rb

@@ -0,0 +1,51 @@
+# QueryReviewer
+require "ostruct"
+require 'erb'
+require 'yaml'
+
+module QueryReviewer
+  CONFIGURATION = {}
+    
+  def self.load_configuration
+    default_config = YAML::load(ERB.new(IO.read(File.join(File.dirname(__FILE__), "..", "query_reviewer_defaults.yml"))).result)
+    
+    CONFIGURATION.merge!(default_config["all"] || {})
+    CONFIGURATION.merge!(default_config[RAILS_ENV || "test"] || {})
+    
+    app_config_file = File.join(RAILS_ROOT, "config", "query_reviewer.yml")
+        
+    if File.exist?(app_config_file)
+      app_config = YAML.load(ERB.new(IO.read(app_config_file)).result)
+      CONFIGURATION.merge!(app_config["all"] || {}) 
+      CONFIGURATION.merge!(app_config[RAILS_ENV || "test"] || {}) 
+    end
+    
+    if enabled?
+      begin      
+        CONFIGURATION["uv"] ||= !Gem.searcher.find("uv").nil?
+        if CONFIGURATION["uv"]
+          require "uv"
+        end
+      rescue
+        CONFIGURATION["uv"] ||= false    
+      end
+    end    
+  end
+  
+  def self.enabled?
+    CONFIGURATION["enabled"]
+  end
+end
+
+QueryReviewer.load_configuration
+
+if QueryReviewer.enabled?
+  require "query_reviewer/query_warning"
+  require "query_reviewer/array_extensions"
+  require "query_reviewer/sql_query"
+  require "query_reviewer/mysql_analyzer"
+  require "query_reviewer/sql_sub_query"
+  require "query_reviewer/mysql_adapter_extensions"
+  require "query_reviewer/controller_extensions"
+  require "query_reviewer/sql_query_collection"
+end

data/lib/query_reviewer/array_extensions.rb

@@ -0,0 +1,29 @@
+module QueryReviewer
+  module ArrayExtensions  #taken from query_analyser plugin
+    protected
+      def qa_columnized_row(fields, sized)
+        row = []
+        fields.each_with_index do |f, i|
+          row << sprintf("%0-#{sized[i]}s", f.to_s)
+        end
+        row.join(' | ')
+      end
+
+    public
+
+    def qa_columnized
+      sized = {}
+      self.each do |row|
+        row.values.each_with_index do |value, i|
+          sized[i] = [sized[i].to_i, row.keys[i].length, value.to_s.length].max
+        end
+      end
+
+      table = []
+      table << qa_columnized_row(self.first.keys, sized)
+      table << '-' * table.first.length
+      self.each { |row| table << qa_columnized_row(row.values, sized) }
+      table.join("\n   ") # Spaces added to work with format_log_entry
+    end
+  end  
+end

data/lib/query_reviewer/controller_extensions.rb

@@ -0,0 +1,60 @@
+require File.join(File.dirname(__FILE__), "views", "query_review_box_helper")
+
+module QueryReviewer
+  module ControllerExtensions
+    class QueryViewBase < ActionView::Base
+      include QueryReviewer::Views::QueryReviewBoxHelper
+    end
+
+    def self.included(base)
+      base.alias_method_chain :perform_action, :query_review if QueryReviewer::CONFIGURATION["inject_view"]
+      base.alias_method_chain :process, :query_review
+      base.helper_method :query_review_output
+    end
+
+    def query_review_output(ajax = false, total_time = nil)
+      faux_view = QueryViewBase.new([File.join(File.dirname(__FILE__), "views")], {}, self)
+      queries = Thread.current["queries"]
+      queries.analyze!
+      faux_view.instance_variable_set("@queries", queries)
+      faux_view.instance_variable_set("@total_time", total_time)
+      if ajax
+        js = faux_view.render(:partial => "/box_ajax.js")
+      else
+        html = faux_view.render(:partial => "/box")
+      end
+    end
+
+    def add_query_output_to_view(total_time)
+      if request.xhr?
+        if cookies["query_review_enabled"]
+          if !response.content_type || response.content_type.include?("text/html")
+            response.body += "<script type=\"text/javascript\">"+query_review_output(true, total_time)+"</script>"
+          elsif response.content_type && response.content_type.include?("text/javascript")
+            response.body += ";\n"+query_review_output(true, total_time)
+          end
+        end
+      else
+        if response.body.is_a?(String) && response.body.match(/<\/body>/i) && Thread.current["queries"]
+          idx = (response.body =~ /<\/body>/i)
+          html = query_review_output(false, total_time)
+          response.body.insert(idx, html)
+        end
+      end
+    end
+
+    def perform_action_with_query_review
+      Thread.current["query_reviewer_enabled"] = cookies["query_review_enabled"]
+      t1 = Time.now
+      r = perform_action_without_query_review
+      t2 = Time.now
+      add_query_output_to_view(t2 - t1)
+      r
+    end
+
+    def process_with_query_review(request, response, method = :perform_action, *arguments) #:nodoc:
+      Thread.current["queries"] = SqlQueryCollection.new
+      process_without_query_review(request, response, method, *arguments)
+    end
+  end
+end

data/lib/query_reviewer/mysql_adapter_extensions.rb

@@ -0,0 +1,90 @@
+module QueryReviewer
+  module MysqlAdapterExtensions
+    def self.included(base)
+      base.alias_method_chain :select, :review
+      base.alias_method_chain :update, :review
+      base.alias_method_chain :insert, :review
+      base.alias_method_chain :delete, :review
+    end
+    
+    def update_with_review(sql, *args)
+      t1 = Time.now
+      result = update_without_review(sql, *args)
+      t2 = Time.now
+
+      create_or_add_query_to_query_reviewer!(sql, nil, t2 - t1, nil, "UPDATE", result)
+
+      result
+    end
+
+    def insert_with_review(sql, *args)
+      t1 = Time.now
+      result = insert_without_review(sql, *args)
+      t2 = Time.now
+
+      create_or_add_query_to_query_reviewer!(sql, nil, t2 - t1, nil, "INSERT")
+
+      result
+    end
+
+    def delete_with_review(sql, *args)
+      t1 = Time.now
+      result = delete_without_review(sql, *args)
+      t2 = Time.now
+
+      create_or_add_query_to_query_reviewer!(sql, nil, t2 - t1, nil, "DELETE", result)
+
+      result
+    end
+    
+    def select_with_review(sql, *args)
+      sql.gsub!(/^SELECT /i, "SELECT SQL_NO_CACHE ") if QueryReviewer::CONFIGURATION["disable_sql_cache"]
+      @logger.silence { execute("SET PROFILING=1") } if QueryReviewer::CONFIGURATION["profiling"]
+      t1 = Time.now
+      query_results = select_without_review(sql, *args)
+      t2 = Time.now
+
+      if @logger && sql =~ /^select/i && query_reviewer_enabled?
+        use_profiling = QueryReviewer::CONFIGURATION["profiling"]
+        use_profiling &&= (t2 - t1) >= QueryReviewer::CONFIGURATION["warn_duration_threshold"].to_f / 2.0 if QueryReviewer::CONFIGURATION["production_data"]
+
+        if use_profiling
+          t5 = Time.now
+          @logger.silence { execute("SET PROFILING=1") }
+          t3 = Time.now
+          select_without_review(sql, *args)
+          t4 = Time.now
+          profile = @logger.silence { select_without_review("SHOW PROFILE ALL", *args) }
+          @logger.silence { execute("SET PROFILING=0") }
+          t6 = Time.now
+          Thread.current["queries"].overhead_time += t6 - t5
+        else
+          profile = nil
+        end
+
+        cols = @logger.silence do
+          select_without_review("explain #{sql}", *args)
+        end
+
+        duration = t3 ? [t2 - t1, t4 - t3].min : t2 - t1
+        create_or_add_query_to_query_reviewer!(sql, cols, duration, profile)
+
+        #@logger.debug(format_log_entry("Analyzing #{name}\n", query.to_table)) if @logger.level <= Logger::INFO
+      end
+      query_results
+    end
+    
+    def query_reviewer_enabled?
+      Thread.current["queries"] && Thread.current["queries"].respond_to?(:find_or_create_sql_query) && Thread.current["query_reviewer_enabled"]
+    end
+    
+    def create_or_add_query_to_query_reviewer!(sql, cols, run_time, profile, command = "SELECT", affected_rows = 1)
+      if query_reviewer_enabled?
+        t1 = Time.now
+        Thread.current["queries"].find_or_create_sql_query(sql, cols, run_time, profile, command, affected_rows)
+        t2 = Time.now
+        Thread.current["queries"].overhead_time += t2 - t1
+      end
+    end
+  end
+end

data/lib/query_reviewer/mysql_analyzer.rb

@@ -0,0 +1,62 @@
+module QueryReviewer
+  module MysqlAnalyzer
+    def do_mysql_analysis!
+      analyze_select_type!
+      analyze_query_type!
+      analyze_key!
+      analyze_extras!
+      analyze_keylen!
+    end
+
+    def analyze_select_type!
+      if select_type.match /uncacheable subquery/
+        warn(:severity => 10, :field => "select_type", :desc => "Subquery must be run once for EVERY row in main query")
+      elsif select_type.match /dependent/
+        warn(:severity => 2, :field => "select_type", :desc => "Dependent subqueries can not be executed while the main query is running")
+      end
+    end
+
+    def analyze_query_type!
+      case query_type
+      when "system", "const", "eq_ref":
+        praise("Yay")
+      when "ref", "ref_or_null", "range", "index_merge":
+        praise("Not bad eh...")
+      when "unique_subquery", "index_subquery":
+        #NOT SURE
+      when "index":
+        warn(:severity => 8, :field => "query_type", :desc => "Full index tree scan (slightly faster than a full table scan)") unless !extra.include?("using where")
+      when "all":
+        warn(:severity => 9, :field => "query_type", :desc => "Full table scan") unless !extra.include?("using where")
+      end
+    end
+
+    def analyze_key!
+      if self.key == "const"
+        praise "Way to go!"
+      elsif self.key.blank? && !self.extra.include?("select tables optimized away")
+        warn :severity => 6, :field => "key", :desc => "No index was used here. In this case, that meant scanning #{self.rows} rows."
+      end
+    end
+
+    def analyze_extras!
+      if self.extra.match(/range checked for each record/)
+        warn :severity => 4, :problem => "Range checked for each record", :desc => "MySQL found no good index to use, but found that some of indexes might be used after column values from preceding tables are known"
+      end
+
+      if self.extra.match(/using filesort/)
+        warn :severity => 2, :problem => "Using filesort", :desc => "MySQL must do an extra pass to find out how to retrieve the rows in sorted order."
+      end
+
+      if self.extra.match(/using temporary/)
+        warn :severity => 10, :problem => "Using temporary table", :desc => "To resolve the query, MySQL needs to create a temporary table to hold the result."
+      end
+    end
+
+    def analyze_keylen!
+      if self.key_len && !self.key_len.to_i.nil? && (self.key_len.to_i > QueryReviewer::CONFIGURATION["max_safe_key_length"])
+        warn :severity => 4, :problem => "Long key length (#{self.key_len.to_i})", :desc => "The key used for the index was rather long, potentially affecting indices in memory"
+      end
+    end
+  end
+end

data/lib/query_reviewer/query_warning.rb

@@ -0,0 +1,17 @@
+module QueryReviewer
+  class QueryWarning
+    attr_reader :query, :severity, :problem, :desc, :table, :id
+
+    cattr_accessor :next_id
+    self.next_id = 1
+    
+    def initialize(options)
+      @query = options[:query]
+      @severity = options[:severity]
+      @problem = options[:problem]
+      @desc = options[:desc]
+      @table = options[:table]
+      @id = (self.class.next_id += 1)
+    end
+  end
+end

data/lib/query_reviewer/sql_query.rb

@@ -0,0 +1,130 @@
+require "ostruct"
+
+module QueryReviewer
+  # a single SQL SELECT query
+  class SqlQuery
+    attr_reader :sqls, :rows, :subqueries, :trace, :id, :command, :affected_rows, :profiles, :durations, :sanitized_sql
+
+    cattr_accessor :next_id
+    self.next_id = 1
+
+    def initialize(sql, rows, full_trace, duration = 0.0, profile = nil, command = "SELECT", affected_rows = 1, sanitized_sql = nil)
+      @trace = full_trace
+      @rows = rows
+      @sqls = [sql]
+      @sanitized_sql = sanitized_sql
+      @subqueries = rows ? rows.collect{|row| SqlSubQuery.new(self, row)} : []
+      @id = (self.class.next_id += 1)
+      @profiles = profile ? [profile.collect { |p| OpenStruct.new(p) }] : [nil]
+      @durations = [duration.to_f]
+      @warnings = []
+      @command = command
+      @affected_rows = affected_rows
+    end
+    
+    def add(sql, duration, profile)
+      sql << sql
+      durations << duration
+      profiles << profile
+    end
+    
+    def sql
+      sqls.first
+    end
+    
+    def count
+      durations.size
+    end
+    
+    def profile
+      profiles.first
+    end
+    
+    def duration
+      durations.sum
+    end
+
+    def duration_stats
+      "TOTAL:#{'%.3f' % duration}  AVG:#{'%.3f' % (durations.sum / durations.size)}  MAX:#{'%.3f' % (durations.max)}  MIN:#{'%.3f' % (durations.min)}"
+    end
+
+    def to_table
+      rows.qa_columnized
+    end
+
+    def warnings
+      self.subqueries.collect(&:warnings).flatten + @warnings
+    end
+
+    def has_warnings?
+      !self.warnings.empty?
+    end
+
+    def max_severity
+      self.warnings.empty? ? 0 : self.warnings.collect(&:severity).max
+    end
+
+    def table
+      @subqueries.first.table
+    end
+
+    def analyze!
+      self.subqueries.collect(&:analyze!)
+      if duration
+        if duration >= QueryReviewer::CONFIGURATION["critical_duration_threshold"]
+          warn(:problem => "Query took #{duration} seconds", :severity => 9)
+        elsif duration >= QueryReviewer::CONFIGURATION["warn_duration_threshold"]
+          warn(:problem => "Query took #{duration} seconds", :severity => QueryReviewer::CONFIGURATION["critical_severity"])
+        end
+      end
+      
+      if affected_rows >= QueryReviewer::CONFIGURATION["critical_affected_rows"]
+        warn(:problem => "#{affected_rows} rows affected", :severity => 9, :description => "An UPDATE or DELETE query can be slow and lock tables if it affects many rows.")
+      elsif affected_rows >= QueryReviewer::CONFIGURATION["warn_affected_rows"]
+        warn(:problem => "#{affected_rows} rows affected", :severity => QueryReviewer::CONFIGURATION["critical_severity"], :description => "An UPDATE or DELETE query can be slow and lock tables if it affects many rows.")
+      end
+    end
+
+    def to_hash
+      @sql.hash
+    end
+
+    def relevant_trace
+      trace.collect(&:strip).select{|t| t.starts_with?(RAILS_ROOT) &&
+          (!t.starts_with?("#{RAILS_ROOT}/vendor") || QueryReviewer::CONFIGURATION["trace_includes_vendor"]) &&
+          (!t.starts_with?("#{RAILS_ROOT}/lib") || QueryReviewer::CONFIGURATION["trace_includes_lib"]) &&
+          !t.starts_with?("#{RAILS_ROOT}/vendor/plugins/query_reviewer") }
+    end
+
+    def full_trace
+      self.class.generate_full_trace(trace)
+    end
+
+    def warn(options)
+      options[:query] = self
+      options[:table] ||= self.table
+      @warnings << QueryWarning.new(options)
+    end
+
+    def select?
+      self.command == "SELECT"
+    end
+    
+    def self.generate_full_trace(trace = Kernel.caller)
+      trace.collect(&:strip).select{|t| !t.starts_with?("#{RAILS_ROOT}/vendor/plugins/query_reviewer") }
+    end
+    
+    def self.sanitize_strings_and_numbers_from_sql(sql)
+      new_sql = sql.clone
+      new_sql.gsub!(/\b\d+\b/, "N")
+      new_sql.gsub!(/\b0x[0-9A-Fa-f]+\b/, "N")
+      new_sql.gsub!(/''/, "'S'")
+      new_sql.gsub!(/""/, "\"S\"")
+      new_sql.gsub!(/\\'/, "")
+      new_sql.gsub!(/\\"/, "")
+      new_sql.gsub!(/'[^']+'/, "'S'")
+      new_sql.gsub!(/"[^"]+"/, "\"S\"")
+      new_sql
+    end
+  end
+end