shiba 0.1.2 → 0.2.0

data/lib/shiba/output/tags.yaml

@@ -1,4 +1,12 @@
 ---
+fuzzed_data:
+  title: Fuzzed Data
+  summary: Shiba doesn't know the size of <b>{{query.table}}</b>.  For these purposes we set the table size to <b>{{query.table_size}}</b>.
+  description: |
+    We're not sure how much data this table will hold in the future, so we've pretended
+    there's 6000 rows in it.  This can lead to a lot of false positives. To
+    improve results, please give shiba your index statistics.
+  level: info
 possible_key_check:
   title: Oddly reported possible keys
   description: |
@@ -7,18 +15,21 @@ possible_key_check:
     key possible.  Sometimes "possible_keys" will be inaccurate and no keys were possible.
   level: info
 access_type_const:
-  title: Constant Lookup
+  title: One row
+  summary: The database only needs to read a single row from <b>{{query.table}}</b>.
   description: |
     This query selects at *most* one row, which is about as good as things get.
   level: success
 access_type_ref:
-  title: Index Lookup (reference)
+  title: Indexed
+  summary: The database reads {{ formattedCost }} rows in <b>{{ query.table }}</b> via the <b>{{ query.key }}</b> index ({{ key_parts }}).
   description: |
     This query uses an index to find rows that match a single value.  Often this
     has very good performance, but it depends on how many rows match that value.
   level: success
 access_type_range:
-  title: Index Lookup (range)
+  title: Indexed
+  summary: The database uses a "range scan" to read more than {{ formattedCost }} rows in {{ query.table }} via the <b>{{ query.key }}</b> index ({{ key_parts }})
   description: |
     This query uses an index to find rows that match a range of values, for instance
     `WHERE indexed_value in (1,2,5,6)` or `WHERE indexed_value >= 5 AND indexed_value <= 15`.
@@ -27,6 +38,7 @@ access_type_range:
   level: info
 access_type_tablescan:
   title: Table Scan
+  summary: The database reads <b>100%</b> ({{ query.table_size }}) of the rows in <b>{{ query.table }}</b>, skipping any indexes.
   description: |
     This query doesn't use any indexes to find data, meaning this query will need to evaluate
     every single row in the table.  This is about the worst of all possible worlds.
@@ -35,15 +47,24 @@ access_type_tablescan:
     but be aware that if this table is not effectively tiny or constant-sized you're entering
     a world of pain.
   level: danger
-fuzzed_data:
-  title: Guessed Table Size
-  description: |
-    We're not sure how much data this table will hold in the future, so we've pretended
-    there's 6000 rows in it.  This can lead to a lot of false positives. To
-    improve results, please give shiba your index statistics.
-  level: info
 ignored:
-  title: Ignored Query
-  description: |
-    This query matched an "ignore" rule in shiba.yml, so we skipped any further analysis on it.
+  title: Ignored
+  summary: This query matched an "ignore" rule in shiba.yml.  Any further analysis was skipped.
+  description:
   level: info
+index_walk:
+  title: Index Walk
+  description: |
+    This query is sorted in the same order as the index used, which means that the database
+    can read the index and simply pluck {LIMIT} rows out of the index.  It's a very
+    fast way to look up stuff by index.
+  level: success
+retsize_bad:
+  title: Big Results
+  summary: The database returns {{ query.return_size.toLocaleString() }} rows to the client.
+  level: danger
+retsize_good:
+  title: Small Results
+  summary: The database returns {{ query.return_size.toLocaleString() }} row(s) to the client.
+  level: success
+

data/lib/shiba/query_watcher.rb

@@ -1,13 +1,13 @@
 require 'shiba/query'
+require 'shiba/backtrace'
 require 'json'
 require 'rails'
 
 module Shiba
   class QueryWatcher
-    IGNORE = /\.rvm|gem|vendor\/|rbenv|seed|db|shiba|test|spec/
 
     def self.watch(file)
-      new(file).watch
+      new(file).tap { |w| w.watch }
     end
 
     attr_reader :queries
@@ -26,7 +26,7 @@ module Shiba
         if sql.start_with?("SELECT")
           fingerprint = Query.get_fingerprint(sql)
           if !@queries[fingerprint]
-            if lines = app_backtrace
+            if lines = Backtrace.from_app
               @file.puts("#{sql} /*shiba#{lines}*/")
             end
           end
@@ -35,48 +35,5 @@ module Shiba
       end
     end
 
-    protected
-
-    # 8 backtrace lines starting from the app caller, cleaned of app/project cruft.
-    def app_backtrace
-      app_line_idx = caller_locations.index { |line| line.to_s !~ IGNORE }
-      if app_line_idx == nil
-        return
-      end
-
-      caller_locations(app_line_idx+1, 8).map do |loc|
-        line = loc.to_s
-        line.sub!(backtrace_ignore_pattern, '')
-        line
-      end
-    end
-
-    def backtrace_ignore_pattern
-      @roots ||= begin
-        paths = Gem.path
-        paths << Rails.root.to_s if Rails.root
-        paths << repo_root
-        paths << ENV['HOME']
-        paths.uniq!
-        paths.compact!
-        # match and replace longest path first
-        paths.sort_by!(&:size).reverse!
-
-        Regexp.new(paths.map {|r| Regexp.escape(r) }.join("|"))
-      end
-    end
-
-    # /user/git_repo => "/user/git_repo"
-    # /user/not_a_repo => nil
-    def repo_root
-      root = nil
-      Open3.popen3('git rev-parse --show-toplevel') {|_,o,_,_|
-        if root = o.gets
-          root = root.chomp
-        end
-      }
-      root
-    end
-
   end
 end

data/lib/shiba/railtie.rb

@@ -1,20 +1,43 @@
 require 'shiba/query_watcher'
 
 class Shiba::Railtie < Rails::Railtie
+  # Logging is enabled when:
+  #  1. SHIBA_OUT environment variable is set to an existing file path.
+  #  2. RSpec/MiniTest exists, in which case a fallback query log is generated at /tmp
   config.after_initialize do
     begin
-      path = ENV['SHIBA_OUT']
-      next if !path
+      path = ENV['SHIBA_OUT'] || "/tmp/shiba-query.log-#{Time.now.to_i}"
+      f = File.open(path, 'a')
+      watcher = Shiba::QueryWatcher.watch(f)
+      next if watcher.nil?
 
-      if File.exist?(path)
-        f = File.open(path, 'a')
-        Shiba::QueryWatcher.watch(f)
-      else
-        $stderr.puts("Shiba could not open '#{path}' for logging.")
+      at_exit do
+        f.close
+        puts ""
+        cmd = "shiba explain #{database_args} --file #{path}"
+        if ENV['SHIBA_DEBUG']
+          $stderr.puts("running:")
+          $stderr.puts(cmd)
+        end
+        system(cmd)
       end
+
     rescue => e
       $stderr.puts("Shiba failed to load")
       $stderr.puts(e.message, e.backtrace.join("\n"))
     end
   end
-end
+
+  def self.database_args
+    c = ActiveRecord::Base.configurations['test']
+    options = {
+      'host':     c['host'],
+      'database': c['database'],
+      'user':     c['username'],
+      'password': c['password']
+    }
+
+    options.reject { |k,v| v.nil? }.map { |k,v| "--#{k} #{v}" }.join(" ")
+  end
+
+end

data/lib/shiba/table_stats.rb

@@ -0,0 +1,34 @@
+require 'shiba/index_stats'
+require 'shiba/fuzzer'
+
+module Shiba
+  class TableStats
+    def initialize(dump_stats, connection, manual_stats)
+      @dump_stats = Shiba::IndexStats.new(dump_stats)
+      @db_stats = Shiba::Fuzzer.new(connection).fuzz!
+      @manual_stats = Shiba::IndexStats.new(manual_stats)
+    end
+
+
+    def estimate_key(table_name, key, parts)
+      ask_each(:estimate_key, table_name, key, parts)
+    end
+
+    def table_count(table)
+      ask_each(:table_count, table)
+    end
+
+    def fuzzed?(table)
+      !@dump_stats.tables[table] && !@manual_stats.tables[table] && @db_stats.tables[table]
+    end
+
+    private
+    def ask_each(method, *args)
+      [@dump_stats, @db_stats].each do |stat|
+        result = stat.send(method, *args)
+        return result unless result.nil?
+      end
+      nil
+    end
+  end
+end

data/lib/shiba/version.rb

@@ -1,3 +1,3 @@
 module Shiba
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/shiba.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.executables   = ["shiba"]
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "activesupport"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/shiba.yml.example

@@ -0,0 +1,4 @@
+ignore:
+  - "acts_as_searchable.rb#fetch_ranks_and_ids"
+  - "issue_query.rb"
+

data/web/main.css

@@ -12,7 +12,37 @@
   margin: 10px
 }
 
+.sql {
+  font-family: monospace;
+}
+
 .badge  {
-  color: white;
-  margin-right: 10px;
+  color: black;
+  background-color: white;
+  border-style: solid;
+  border-width: 2px;
+  margin-right: 5px;
+  width: 100px;
+}
+
+.shiba-badge-info {
+  border-color: #78b0ec;
+}
+
+.shiba-badge-danger {
+  border-color: #ff655d;
+}
+
+.shiba-badge-success {
+  border-color: green;
+}
+
+.shiba-badge-warning {
+  border-color: #ffb100;
+}
+
+.shiba-info-list {
+  list-style-type:none;
+  margin: 5px;
+  padding: 0;
 }

data/web/results.html.erb

@@ -1,44 +1,84 @@
 <html>
   <head>
     <title>Shiba results for <%= Time.now %></title>
-    <% data[:js].each do |js| %>
-      <script type="text/javascript" src="./js/<%= js %>"></script>
-    <% end %>
-    <link rel="stylesheet" href="bootstrap.min.css">
-    <link rel="stylesheet" href="main.css">
     <link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900|Material+Icons" rel="stylesheet">
   </head>
   <body>
+    <% data[:js].each do |js| %>
+    <script type="text/javascript">
+      <%= File.read(js) %>
+    </script>
+    <% end %>
+
+    <% data[:css].each do |css| %>
+      <style type="text/css">
+        <%= File.read(css) %>
+      </style>
+    <% end %>
+
     <script language="javascript">
       var data = <%= data.to_json %>;
       var queriesByTable = [];
       var queriesByTableLow = [];
+      var queriesHaveFuzzed = false;
+      var severityIndexes = { high: 1, medium: 2, low: 3 };
 
-
-      function sortByTable(a, b) {
-        if ( a.table < b.table )
-          return -1;
-        else if ( b.table < a.table )
-          return 1;
-        else {
-          return b.cost - a.cost;
+      function sortByFunc(fields) {
+        return function(a, b) {
+          for ( var i = 0 ; i < fields.length; i++ ) {
+            if ( a[fields[i]] < b[fields[i]] )
+              return -1;
+            else if ( a[fields[i]] > b[fields[i]] )
+              return 1;
+          }
+          return 0;
         }
       }
 
+      function Query(obj) {
+        Object.assign(this, obj);
+        this.severityIndex = severityIndexes[this.severity];
+        this.splitSQL();
+      };
+
+      Query.prototype = {
+        hasTag: function(tag) {
+          this.tags.includes(tag);
+        },
+        splitSQL: function() {
+          this.sqlFragments = this.sql.match(/(SELECT\s)(.*?)(\s+FROM .*)/i);
+        },
+        select: function () {
+          return this.sqlFragments[1];
+        },
+        fields: function () {
+          return this.sqlFragments[2];
+        },
+        rest: function(index) {
+          return this.sqlFragments.slice(index).join('');
+        }
+      };
+
       data.queries.forEach(function(query) {
-        if ( query.cost < 100 ) {
-          queriesByTableLow.push(query);
+        var q = new Query(query);
+
+        if ( q.cost < 100 ) {
+          queriesByTableLow.push(q);
         } else {
-          queriesByTable.push(query);
+          queriesByTable.push(q);
         }
 
-        if ( !query.tags.includes("fuzzed_data") )
-          query.costAvailable = true;
+        if ( q.hasTag("fuzzed_data" ) )
+          queriesHaveFuzzed = true;
+
+        q.expandSelect = false;
       });
 
-      queriesByTable = queriesByTable.sort(sortByTable);
-      queriesByTableLow = queriesByTableLow.sort(sortByTable);
+      var f = sortByFunc(['severityIndex', 'table']);
+      queriesByTable = queriesByTable.sort(f);
+      queriesByTableLow = queriesByTableLow.sort(f);
     </script>
+
     <script type="text/x-template" id="query-template">
       <div class="query">
         <div class="row">
@@ -50,38 +90,94 @@
           </div>
           <div class="col-5">{{ truncate(query.sql, 50) }}</div>
           <div class="col-3" v-html="makeURL(query.backtrace[0], shortLocation(query))"></div>
-          <div class="col-1">{{ severity(query) }}</div>
+          <div class="col-1">{{ query.severity }}</div>
         </div>
         <div class="row" v-if="expanded">
           <div class="col-12">
             <div class="query-info-box">
-              <pre style="white-space: pre-wrap;">{{ query.sql }}</pre>
+              <query-sql v-bind:query="query"></query-sql>
               <div v-if="query.backtrace && query.backtrace.length > 0">
                 Stack Trace:<br>
                 <div class="backtrace">
                   <div v-for="backtrace in query.backtrace" v-html="makeURL(backtrace, backtrace)"></div>
                 </div>
               </div>
-              Key: {{ castNull(query.key) }}<br>
-              <div v-if="query.possible_keys && query.possible_keys.length > 0 || !query.key">
-                Possible keys: {{ castNull(query.possible_keys) }}
+              <ul class="shiba-info-list">
+              <li v-for="tag in query.tags">
+                <component v-bind:is="'tag-' + tag" v-bind:query="query"></component>
+              </li>
+              </ul>
+              <div v-if="!rawExpanded">
+                <a href="#" v-on:click.prevent="rawExpanded = !rawExpanded">See full EXPLAIN</a>
+              </div>
+              <div v-else>
+                <a href="#" v-on:click.prevent="rawExpanded = !rawExpanded">hide EXPLAIN</a>
+                <pre class="backtrace">{{  JSON.stringify(query.raw_explain, null, 2) }}</pre>
               </div>
-              Cost: {{ query.cost }}<br>
-              <a class="badge"
-                v-for="tag in query.tags"
-                href="#"
-                v-bind:class="tagClass(tag)"
-                v-on:click="expandInfo(tag, $event)">{{ tagTitle(tag) }}</a>
             </div>
           </div>
         </div>
       </div>
     </script>
 
+    <% data[:tags].each do |tag, h| %>
+    <script type="text/x-template" id="tag-<%= tag %>-template">
+      <span><a class="badge shiba-badge-<%= h['level'] %>"><%= h['title'] %></a><%= h['summary'] %></span>
+    </script>
+    <script>
+      Vue.component('tag-<%= tag %>', {
+        template: '#tag-<%= tag %>-template',
+        props: ['query'],
+        computed: {
+          key_parts: function() {
+            if ( this.query.used_key_parts && this.query.used_key_parts.length > 0 )
+              return this.query.used_key_parts.join(',');
+            else
+              return "";
+          },
+          formattedCost: function() {
+            var costPercentage = (this.query.cost / this.query.table_size) * 100.0;
+            if ( this.query.cost > 100 && costPercentage > 1 ) // todo: make better
+              return `${costPercentage.toFixed()}% (${this.query.cost.toLocaleString()}) of the`;
+            else
+              return this.query.cost.toLocaleString();
+          }
+        }
+      });
+    </script>
+    <% end %>
+
+
+    <script type="text/x-template" id="sql-template">
+      <div class="sql">
+        <span>{{ query.select() }}</span>
+        <span v-if="!expandFields && query.fields().length > 80">
+          <a href="#" v-on:click.prevent="expandFields = !expandFields">...</a>
+        </span>
+        <span v-else>{{ query.fields() }}</span>
+        <span>{{ query.rest(3) }}</span>
+      </div>
+    </script>
+
+    <script>
+      Vue.component('query-sql', {
+        template: '#sql-template',
+        props: ['query'],
+        data: function () {
+          return { expandFields: false }
+        }
+      });
+    </script>
+
     <div id="app">
       <v-dialog :width="600"></v-dialog>
       <div class="container" style="">
         <div class="row">
+          <div v-if="hasFuzzed" class="alert alert-warning" role="alert">
+            This query analysis was generated using estimated table sizes.
+            <a href="https://github.com/burrito-brothers/shiba/blob/master/README.md#going-beyond-table-scans">Find out how to get a more accurate analysis by feeding Shiba index stats</a>
+          </div>
+
           <div class="col-12">We found {{ queries.length }} queries that deserve your attention: </div>
         </div>
         <div class="row">
@@ -110,7 +206,8 @@
       props: ['query', 'tags', 'github'],
       data: function () {
         return {
-          expanded: false
+          expanded: false,
+          rawExpanded: false
         };
       },
       methods: {
@@ -137,18 +234,6 @@
           if (event) event.preventDefault()
           this.expanded = !this.expanded;
         },
-        severity: function(query) {
-          if ( query.cost > 1 )
-            return "high";
-          else
-            return "low";
-        },
-        castNull: function (string) {
-          if ( !string )
-            return "(none)";
-          else
-            return string;
-        },
         shortLocation: function(query) {
           if ( !query.backtrace )
             return null;
@@ -164,24 +249,12 @@
           var line = matches[2];
 
           return `<a href='${data.url}/${file}#L${line}' target='_new'>${content}</a>`;
-        },
-        tagTitle: function(tag) {
-          if ( !this.tags[tag] )
-            return tag;
-          else
-            return this.tags[tag].title;
-        },
-        tagClass: function(tag) {
-          if ( !this.tags[tag] )
-            return '';
-          else
-            return "badge-" + this.tags[tag].level;
         }
       },
       computed: {
         expandText: function() {
           return this.expanded ? "-" : "+";
-        },
+        }
       }
     });
 
@@ -191,7 +264,8 @@
         queries: queriesByTable,
         queriesLow: queriesByTableLow,
         tags: data.tags,
-        lowExpanded: false
+        lowExpanded: false,
+        hasFuzzed: queriesHaveFuzzed
       },
       methods: { }
     });