lookout-query_reviewer 0.1.5

data/lib/query_reviewer/query_warning.rb

@@ -0,0 +1,17 @@
+module QueryReviewer
+  class QueryWarning
+    attr_reader :query, :severity, :problem, :desc, :table, :id
+
+    cattr_accessor :next_id
+    self.next_id = 1
+    
+    def initialize(options)
+      @query = options[:query]
+      @severity = options[:severity]
+      @problem = options[:problem]
+      @desc = options[:desc]
+      @table = options[:table]
+      @id = (self.class.next_id += 1)
+    end
+  end
+end

data/lib/query_reviewer/rails.rb

@@ -0,0 +1,37 @@
+require 'query_reviewer'
+
+module QueryReviewer
+  def self.inject_reviewer
+    # Load adapters
+    ActiveRecord::Base
+    adapter_class = ActiveRecord::ConnectionAdapters::MysqlAdapter  if defined? ActiveRecord::ConnectionAdapters::MysqlAdapter
+    adapter_class = ActiveRecord::ConnectionAdapters::Mysql2Adapter if defined? ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    adapter_class.send(:include, QueryReviewer::MysqlAdapterExtensions) if adapter_class
+    # Load into controllers
+    ActionController::Base.send(:include, QueryReviewer::ControllerExtensions)
+    Array.send(:include, QueryReviewer::ArrayExtensions)
+    if ActionController::Base.respond_to?(:append_view_path)
+      ActionController::Base.append_view_path(File.dirname(__FILE__) + "/lib/query_reviewer/views")
+    end
+  end
+end
+
+if defined?(Rails::Railtie)
+  module QueryReviewer
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        load File.dirname(__FILE__) + "/tasks.rb"
+      end
+
+      initializer "query_reviewer.initialize" do
+        QueryReviewer.load_configuration
+
+        QueryReviewer.inject_reviewer if QueryReviewer.enabled?
+      end
+    end
+  end
+else # Rails 2
+  QueryReviewer.load_configuration
+
+  QueryReviewer.inject_reviewer if QueryReviewer.enabled?
+end

data/lib/query_reviewer/sql_query.rb

@@ -0,0 +1,131 @@
+require "ostruct"
+
+module QueryReviewer
+  # a single SQL SELECT query
+  class SqlQuery
+    attr_reader :sqls, :rows, :subqueries, :trace, :id, :command, :affected_rows, :profiles, :durations, :sanitized_sql
+
+    cattr_accessor :next_id
+    self.next_id = 1
+
+    def initialize(sql, rows, full_trace, duration = 0.0, profile = nil, command = "SELECT", affected_rows = 1, sanitized_sql = nil)
+      @trace = full_trace
+      @rows = rows
+      @sqls = [sql]
+      @sanitized_sql = sanitized_sql
+      @subqueries = rows ? rows.collect{|row| SqlSubQuery.new(self, row)} : []
+      @id = (self.class.next_id += 1)
+      @profiles = profile ? [profile.collect { |p| OpenStruct.new(p) }] : [nil]
+      @durations = [duration.to_f]
+      @warnings = []
+      @command = command
+      @affected_rows = affected_rows
+    end
+
+    def add(sql, duration, profile)
+      sql << sql
+      durations << duration
+      profiles << profile
+    end
+
+    def sql
+      sqls.first
+    end
+
+    def count
+      durations.size
+    end
+
+    def profile
+      profiles.first
+    end
+
+    def duration
+      durations.sum
+    end
+
+    def duration_stats
+      "TOTAL:#{'%.3f' % duration}  AVG:#{'%.3f' % (durations.sum / durations.size)}  MAX:#{'%.3f' % (durations.max)}  MIN:#{'%.3f' % (durations.min)}"
+    end
+
+    def to_table
+      rows.qa_columnized
+    end
+
+    def warnings
+      self.subqueries.collect(&:warnings).flatten + @warnings
+    end
+
+    def has_warnings?
+      !self.warnings.empty?
+    end
+
+    def max_severity
+      self.warnings.empty? ? 0 : self.warnings.collect(&:severity).max
+    end
+
+    def table
+      @subqueries.first.table
+    end
+
+    def analyze!
+      self.subqueries.collect(&:analyze!)
+      if duration
+        if duration >= QueryReviewer::CONFIGURATION["critical_duration_threshold"]
+          warn(:problem => "Query took #{duration} seconds", :severity => 9)
+        elsif duration >= QueryReviewer::CONFIGURATION["warn_duration_threshold"]
+          warn(:problem => "Query took #{duration} seconds", :severity => QueryReviewer::CONFIGURATION["critical_severity"])
+        end
+      end
+
+      if affected_rows >= QueryReviewer::CONFIGURATION["critical_affected_rows"]
+        warn(:problem => "#{affected_rows} rows affected", :severity => 9, :description => "An UPDATE or DELETE query can be slow and lock tables if it affects many rows.")
+      elsif affected_rows >= QueryReviewer::CONFIGURATION["warn_affected_rows"]
+        warn(:problem => "#{affected_rows} rows affected", :severity => QueryReviewer::CONFIGURATION["critical_severity"], :description => "An UPDATE or DELETE query can be slow and lock tables if it affects many rows.")
+      end
+    end
+
+    def to_hash
+      @sql.hash
+    end
+
+    def relevant_trace
+      trace.collect(&:strip).select{|t| t.starts_with?(Rails.root.to_s) &&
+          (!t.starts_with?("#{Rails.root}/vendor") || QueryReviewer::CONFIGURATION["trace_includes_vendor"]) &&
+          (!t.starts_with?("#{Rails.root}/lib") || QueryReviewer::CONFIGURATION["trace_includes_lib"]) &&
+          !t.starts_with?("#{Rails.root}/vendor/plugins/query_reviewer") }
+    end
+
+    def full_trace
+      self.class.generate_full_trace(trace)
+    end
+
+    def warn(options)
+      options[:query] = self
+      options[:table] ||= self.table
+      @warnings << QueryWarning.new(options)
+    end
+
+    def select?
+      self.command == "SELECT"
+    end
+
+    def self.generate_full_trace(trace = Kernel.caller)
+      trace.collect(&:strip).select{|t| !t.starts_with?("#{Rails.root}/vendor/plugins/query_reviewer") }
+    end
+
+    def self.sanitize_strings_and_numbers_from_sql(sql)
+      new_sql = sql.clone
+      new_sql = new_sql.to_sql if new_sql.respond_to?(:to_sql)
+      new_sql.gsub!(/\b\d+\b/, "N")
+      new_sql.gsub!(/\b0x[0-9A-Fa-f]+\b/, "N")
+      new_sql.gsub!(/''/, "'S'")
+      new_sql.gsub!(/""/, "\"S\"")
+      new_sql.gsub!(/\\'/, "")
+      new_sql.gsub!(/\\"/, "")
+      new_sql.gsub!(/'[^']+'/, "'S'")
+      new_sql.gsub!(/"[^"]+"/, "\"S\"")
+      new_sql
+    end
+  end
+end

data/lib/query_reviewer/sql_query_collection.rb

@@ -0,0 +1,103 @@
+module QueryReviewer
+  # a collection of SQL SELECT queries  
+  class SqlQueryCollection
+    COMMANDS = %w(SELECT DELETE INSERT UPDATE)
+
+    attr_reader :query_hash
+    attr_accessor :overhead_time
+    def initialize(query_hash = {})
+      @query_hash = query_hash
+      @overhead_time = 0.0
+    end
+    
+    def queries
+      query_hash.values
+    end
+
+    def total_duration
+      self.queries.collect(&:durations).flatten.sum
+    end
+
+    def query_count
+      queries.collect(&:count).sum
+    end
+    
+    def analyze!
+      self.queries.collect(&:analyze!)
+
+      @warnings = []
+
+      crit_severity = 9# ((QueryReviewer::CONFIGURATION["critical_severity"] + 10)/2).to_i
+      warn_severity = QueryReviewer::CONFIGURATION["critical_severity"] - 1 # ((QueryReviewer::CONFIGURATION["warn_severity"] + QueryReviewer::CONFIGURATION["critical_severity"])/2).to_i
+
+      COMMANDS.each do |command|
+        count = count_of_command(command)
+        if count > QueryReviewer::CONFIGURATION["critical_#{command.downcase}_count"]
+          warn(:severity => crit_severity, :problem => "#{count} #{command} queries on this page", :description => "Too many #{command} queries can severely slow down a page")
+        elsif count > QueryReviewer::CONFIGURATION["warn_#{command.downcase}_count"]
+          warn(:severity => warn_severity, :problem => "#{count} #{command} queries on this page", :description => "Too many #{command} queries can slow down a page")
+        end
+      end
+    end
+    
+    def find_or_create_sql_query(sql, cols, run_time, profile, command, affected_rows)
+      sanitized_sql = SqlQuery.sanitize_strings_and_numbers_from_sql(sql)
+      trace = SqlQuery.generate_full_trace(Kernel.caller)
+      key = [sanitized_sql, trace]
+      if query_hash[key]
+        query_hash[key].add(sql, run_time, profile)
+      else
+        query_hash[key] = SqlQuery.new(sql, cols, trace, run_time, profile, command, affected_rows, sanitized_sql)
+      end
+    end
+    
+    def warn(options)
+      @warnings << QueryWarning.new(options)
+    end
+    
+    def warnings
+      self.queries.collect(&:warnings).flatten.sort{|a,b| b.severity <=> a.severity}
+    end
+
+    def without_warnings
+      self.queries.reject{|q| q.has_warnings?}.sort{|a,b| b.duration <=> a.duration}
+    end
+    
+    def collection_warnings
+      @warnings
+    end
+    
+    def max_severity
+      warnings.empty? && collection_warnings.empty? ? 0 : [warnings.empty? ? 0 : warnings.collect(&:severity).flatten.max, collection_warnings.empty? ? 0 : collection_warnings.collect(&:severity).flatten.max].max
+    end
+
+    def only_of_command(command, only_no_warnings = false)
+      qs = only_no_warnings ? self.without_warnings : self.queries
+      qs.select{|q| q.command == command}
+    end
+
+    def count_of_command(command, only_no_warnings = false)
+      only_of_command(command, only_no_warnings).collect(&:durations).collect(&:size).sum
+    end
+    
+    def total_severity
+      warnings.collect(&:severity).sum
+    end
+    
+    def total_with_warnings
+      queries.select(&:has_warnings?).length
+    end
+
+    def total_without_warnings
+      queries.length - total_with_warnings
+    end
+    
+    def percent_with_warnings
+      queries.empty? ? 0 : (100.0 * total_with_warnings / queries.length).to_i
+    end
+
+    def percent_without_warnings
+      queries.empty? ? 0 : (100.0 * total_without_warnings / queries.length).to_i
+    end
+  end
+end

data/lib/query_reviewer/sql_sub_query.rb

@@ -0,0 +1,45 @@
+module QueryReviewer
+  # a single part of an SQL SELECT query
+  class SqlSubQuery < OpenStruct
+    include MysqlAnalyzer
+
+    delegate :sql, :to => :parent
+    attr_reader :cols, :warnings, :parent
+    def initialize(parent, cols)
+      @parent = parent
+      @warnings = []
+      @cols = cols.inject({}) {|memo, obj| memo[obj[0].to_s.downcase] = obj[1].to_s.downcase; memo }
+      @cols["query_type"] = @cols.delete("type")
+      super(@cols)
+    end
+
+    def analyze!
+      @warnings = []
+      adapter_name = ActiveRecord::Base.connection.instance_variable_get("@config")[:adapter]
+      adapter_name = 'mysql' if adapter_name == 'mysql2'
+      method_name = "do_#{adapter_name}_analysis!"
+      self.send(method_name.to_sym)
+    end
+
+    def table
+      @table[:table]
+    end
+
+    private
+
+    def warn(options)
+      if (options[:field])
+        field = options.delete(:field)
+        val = self.send(field)
+        options[:problem] = ("#{field.to_s.titleize}: #{val.blank? ? "(blank)" : val}")
+      end
+      options[:query] = self
+      options[:table] = self.table
+      @warnings << QueryWarning.new(options)
+    end
+
+    def praise(options)
+      # no credit, only pain
+    end
+  end
+end

data/lib/query_reviewer/tasks.rb

@@ -0,0 +1,8 @@
+namespace :query_reviewer do
+  desc "Create a default config/query_reviewer.yml"
+  task :setup do
+    defaults_path = File.join(File.dirname(__FILE__), "../..", "query_reviewer_defaults.yml")
+    dest_path = File.join(Rails.root.to_s, "config", "query_reviewer.yml")
+    FileUtils.copy(defaults_path, dest_path)
+  end
+end

data/lib/query_reviewer/views/_box.html.erb

@@ -0,0 +1,11 @@
+<div id="query_review_parent" class="query_review_parent">
+  <div id="query_review_0" class="query_review_container">
+    <%= render :partial => "/box_includes"%>
+    <div class="query_review <%= parent_div_class %>" id = "query_review_header_0">
+      <%= render :partial => "/box_header" %>
+    </div>
+    <div class="query_review_details" id="query_review_details_0" style="display: none;">
+      <%= render :partial => enabled_by_cookie ? "/box_body" : "/box_disabled" %>
+    </div>
+  </div>
+</div>

data/lib/query_reviewer/views/_box_ajax.js

@@ -0,0 +1,34 @@
+var i = 0;
+var last = null;
+for(i=0; i<10; i++) {
+  if(document.getElementById("query_review_"+i)) {
+    last = document.getElementById("query_review_"+i);
+  } else {
+    break;
+  }
+}
+if(i < 10) {
+  var container_div = document.createElement("div");
+  container_div.style.left = ""+(i * 30 + 1)+"px";
+  container_div.setAttribute("id", "query_review_"+i);
+  container_div.className = "query_review_container";
+  var ihtml = '<%= escape_javascript(render(:partial => "/box_includes"))%>';
+  ihtml += '<div class="query_review <%= parent_div_class %>" id = "query_review_header_'+i+'">';
+  ihtml += '<%= escape_javascript(render(:partial => "/box_header"))%>';
+  ihtml += '</div>    ';
+  ihtml += '<div class="query_review_details" id="query_review_details_'+i+'" style="display: none;">';
+  ihtml += '<%= escape_javascript(render(:partial => enabled_by_cookie ? "/box_body" : "/box_disabled")) %>';
+  ihtml += '</div>';
+
+  container_div.innerHTML = ihtml;
+
+  var parent_div = document.getElementById("query_review_parent")
+  if(!parent_div) {
+    parent_div = document.createElement("div");
+    parent_div.setAttribute("id", "query_review_parent");
+    parent_div.className = "query_reivew_parent";
+    document.getElementById("body")[0].appendChild(parent_div);
+  }
+
+  parent_div.appendChild(container_div);
+}

data/lib/query_reviewer/views/_box_body.html.erb

@@ -0,0 +1,73 @@
+    <p>Total queries: <span class="number"><%= @queries.query_count %></span>&nbsp;&nbsp;
+      <% if @total_time %>Total time: <span class="number" title="TOTAL TIME: <%= @total_time %>s  QR_OVERHEAD: <%= @queries.overhead_time %>s">
+        <%= '%.3f' % (@total_time - @queries.overhead_time) %></span>s&nbsp;&nbsp;
+      <% end %>
+      Database Time: <span class="number"><%= '%.3f' % @queries.total_duration %></span>s</p>
+    <p class="indent">With warnings: <span class="number bad"><%= @queries.total_with_warnings %></span> (<%= @queries.percent_with_warnings %>%)</p>
+    <p class="indent">Without warnings: <span class="number good"><%= @queries.total_without_warnings %></span> (<%= @queries.percent_without_warnings %>%)</p>
+  <p>Type:
+      <% QueryReviewer::SqlQueryCollection::COMMANDS.each do |command| %>
+        <% next if @queries.count_of_command(command).zero? %>
+        <span class="number"><%= @queries.count_of_command(command) %></span> <%= command %>s&nbsp;&nbsp;
+      <% end %>
+  </p>
+    <% if warnings_no_query_sorted.length + queries_with_warnings_sorted.length > 0 %>
+        <div class="divider"></div>
+        <% if warnings_no_query_sorted_nonignored.length + queries_with_warnings_sorted_nonignored.length > 0 %>
+            <p class="title"><%= warnings_no_query_sorted_nonignored.length + queries_with_warnings_sorted_nonignored.length %> Errors:</p>
+            <ul>
+                <%= render :partial => "/warning_no_query", :collection => warnings_no_query_sorted_nonignored %>
+                <%= render :partial => "/query_with_warning", :collection => queries_with_warnings_sorted_nonignored %>
+            </ul>
+        <% end %>
+        <% if warnings_no_query_sorted_ignored.length + queries_with_warnings_sorted_ignored.length > 0 %>
+            <%= warnings_no_query_sorted_ignored.length + queries_with_warnings_sorted_ignored.length %> Warnings:
+            <ul id="query_review_ignored_warnings">
+                <%= render :partial => "/warning_no_query", :collection => warnings_no_query_sorted_ignored %>
+                <%= render :partial => "/query_with_warning", :collection => queries_with_warnings_sorted_ignored %>
+            </ul>
+        <% end %>
+    <% end %>
+    <div class="divider"></div>
+    <p class="title">Safe queries:</p>
+    <% if @queries.queries.empty? %>
+        No queries to display.
+    <% else %>
+    <% QueryReviewer::SqlQueryCollection::COMMANDS.reverse.each do |command| %>
+      <% next if @queries.count_of_command(command, true).zero? %>
+      <ul class="small">
+        <% @queries.only_of_command(command, true).each do |query| %>
+        <li>
+          <% if QueryReviewer::CONFIGURATION["production_data"] %>
+            <%= duration_with_color(query) %>s
+          <% end %>
+          <% if query.count > 1 %>
+            <b title="<%= query.count %> queries were executed with the same stack trace and similar SQL structure">
+              <%= query.count  %> identical queries
+            </b>
+          <% end %>
+          <%= render :partial => "/query_sql", :locals=>{ :query_sql => query } %>
+            <% if query.select? %>
+              <a href="#" onclick="query_review_toggle('warning_<%= query.id %>_explain')" title="show/hide sql">EXPLN</a>
+            <% end %>
+            <% if QueryReviewer::CONFIGURATION["profiling"] && query.profile %>
+              <a href="#" onclick="query_review_toggle('warning_<%= query.id %>_profile')" title="show/hide profile">PROF</a>
+            <% end %>
+            <a href="#" onclick="query_review_toggle('warning_<%= query.id %>_trace')" title="show/hide stack trace">TRACE</a>
+            <div style="display: none" id="warning_<%= query.id %>_explain" class="indent small tbpadded">
+              <%= render :partial => "/explain", :locals => {:query => query} %>
+            </div>
+            <% if QueryReviewer::CONFIGURATION["profiling"] && query.profile %>
+              <div style="display: none" id="warning_<%= query.id %>_profile" class="indent small">
+                <%= render :partial => "/profile", :locals => {:query => query} %>
+              </div>
+            <% end %>
+            <div style="display: none" id="warning_<%= query.id %>_trace" class="indent small">
+              <%= render :partial => "/query_trace", :locals => {:query_trace => query.relevant_trace, :query_id => query.id, :full_trace => query.full_trace} %>
+            </div>
+        </li>
+        <% end %>
+      </ul>
+    <% end %>
+  <% end %>
+  <p id="query_review_disable_link"><a href="#" onclick="eraseCookie('query_review_enabled'); query_review_hide('query_review_disable_link'); alert('Cookie successfully set.');">Disable analysis report</a> on next page load and from now on.</p>