shiba 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0eff935dfd9607d514fc16e2615f5af7d415602d
-  data.tar.gz: c7aa551bccf103087df9848638462d8b3b2d0c30
+  metadata.gz: f6436b087e61bcce7488d2fbb04003c3fd32fb36
+  data.tar.gz: 348c1fa32744f1879e489fa5530294a4ea70fe4e
 SHA512:
-  metadata.gz: 6f0d6576e905ced52788b0ea247f7f0d51c5ea997f1064114028ab4cc3f1811b6adf8e75f9f7b97344b58eb2d579953bfab77a32fc6fcae33c755cb8a8832a2a
-  data.tar.gz: a16096e13a86dc7a8b977c21334590f46701bc1ef9e77c06e8a23f334fa36bcad78a52f2716892881c852569232c9fbb79341c93ea3b311867a4da6f7eb8bf6a
+  metadata.gz: e84dcdbd7cb06d3a2dfa7fa87e9c0bf40e8a3059a8396284c8503599d3d94cbb4c45681b081e8074abc700a7285f64239777cd30c5d1f5eed17f6302756ac98a
+  data.tar.gz: be47f54c87cad50e7260d64468b2c0489f55163a57db59173c3302da9e82a9f62380b239a65eecab6d6db849b044b38d1c0821fdb66457d8497d73f29a6b34f0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    shiba (0.1.1)
+    shiba (0.1.2)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -11,17 +11,20 @@ You can run shiba either as a gem in your test suite, or as a standalone utility
 
 ### Gem
 
-Add this line to your application's Gemfile:
+Install
 
 ```ruby
-group :test do
-  gem 'shiba'
-end
+bundle add shiba --group "development, test"
+```
 
-bundle
+Run shiba
 
+```ruby
 # Run some some code using shiba to generate a SQL report
 bundle exec shiba analyze rake test:functional
+
+# Or run a single test
+bundle exec shiba analyze rails test test/controllers/users_controller_test.rb
 ```
 
 ### Standalone:
@@ -50,4 +53,4 @@ local:$ bin/analyze.rb -h 127.0.0.1 -d TESTDB -u MYSQLUSER -p MYSQLPASS -s shiba
 
 local:$ jq -C -s 'sort_by(.cost) | reverse' results.json | less -R
 
-```
+```

data/bin/analyze

@@ -4,55 +4,14 @@
 
 require 'bundler/setup'
 require 'optionparser'
+require 'shiba'
 require 'shiba/configure'
 require 'shiba/output'
 
 options = {}
-parser = OptionParser.new do |opts|
-  opts.banner = "analyze <command>. Creates report of SQL from the running process."
-
-  opts.on("-h","--host HOST", "sql host") do |h|
-    options["host"] = h
-  end
-
-  opts.on("-d","--database DATABASE", "sql database") do |d|
-    options["database"] = d
-  end
-
-  opts.on("-u","--user USER", "sql user") do |u|
-    options["user"] = u
-  end
-
-  opts.on("-p","--password PASSWORD", "sql password") do |p|
-    options["password"] = p
-  end
-
-  opts.on("-i","--index INDEX", "index of query to inspect") do |i|
-    options["index"] = i.to_i
-  end
-
-  opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
-    options["limit"] = l.to_i
-  end
-
-  opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
-    options["stats"] = f
-  end
-
-  opts.on("-f", "--file FILE", "location of file containing queries") do |f|
-    options["file"] = f
-  end
-
-  opts.on("-o", "--output FILE", "write to file instead of stdout") do |f|
-    options["output"] = f
-  end
-
-  opts.on("-t", "--test", "analyze queries at --file instead of analyzing a process") do |f|
-    options["test"] = true
-  end
-
-end
 
+parser = Shiba::Configure.make_options_parser(options)
+parser.banner = "analyze <command>. Creates report of SQL from the running process."
 parser.parse!
 
 if options["test"] && !options["file"]
@@ -63,24 +22,26 @@ end
 
 # Automagic configuration goes here
 if !options["database"]
-    config = Shiba::Configure.activerecord_configuration
-
-    if tc = config && config['test']
-      $stderr.puts "Reading configuration from '#{`pwd`.chomp}/config/database.yml'[:test]."
-      options['host']     ||= tc['host']
-      options['database'] ||= tc['database']
-      options['user'] ||= tc['username']
-      options['password'] ||= tc['password']
-    end
+  config = Shiba::Configure.activerecord_configuration
+
+  if tc = config && config['test']
+    $stderr.puts "Reading configuration from '#{`pwd`.chomp}/config/database.yml'[:test]."
+    options['host']     ||= tc['host']
+    options['database'] ||= tc['database']
+    options['username'] ||= tc['username']
+    options['password'] ||= tc['password']
   end
+end
 
-  if !options["file"]
-    options["file"] = `mktemp /tmp/shiba-query.log-#{Time.now.to_i}`.chomp
-  end
+Shiba.configure(options)
 
-  if !options["output"]
-    options["output"] = `mktemp /tmp/shiba-explain.log-#{Time.now.to_i}`.chomp
-  end
+if !options["file"]
+  options["file"] = `mktemp /tmp/shiba-query.log-#{Time.now.to_i}`.chomp
+end
+
+if !options["explain"]
+  options["explain"] = `mktemp /tmp/shiba-explain.log-#{Time.now.to_i}`.chomp
+end
 
 # Log process queries
 if !options.delete("test")
@@ -105,7 +66,7 @@ if !options.delete("test")
 end
 
 # Explain
-$stderr.puts "Analyzing SQL to '#{options["output"]}'..."
+$stderr.puts "Analyzing SQL to '#{options["explain"]}'..."
 path = "#{File.dirname(__FILE__)}/explain"
 args = options.select { |_,v| !v.nil? }.map { |k,v| [ "--#{k}", v ] }.flatten
 

data/bin/explain

@@ -10,50 +10,9 @@ require 'shiba/output'
 require 'optionparser'
 
 options = {}
-parser = OptionParser.new do |opts|
-  opts.banner = "Usage: explain -h HOST -d DB -u USER -p PASS [-f QUERY_FILE] [-s STATS_FILE] "
-
-  opts.on("-h","--host HOST", "sql host") do |h|
-    options["host"] = h
-  end
-
-  opts.on("-d","--database DATABASE", "sql database") do |d|
-    options["database"] = d
-  end
-
-  opts.on("-u","--user USER", "sql user") do |u|
-    options["username"] = u
-  end
-
-  opts.on("-p","--password PASSWORD", "sql password") do |p|
-    options["password"] = p
-  end
-
-  opts.on("-i","--index INDEX", "index of query to inspect") do |i|
-    options["index"] = i.to_i
-  end
-
-  opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
-    options["limit"] = l.to_i
-  end
-
-  opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
-    options["stats_file"] = f
-  end
-
-  opts.on("-f", "--file FILE", "location of file containing queries") do |f|
-    options["file"] = f
-  end
-
-  opts.on("-o", "--output FILE", "write to file instead of stdout") do |f|
-    options["output"] = f
-  end
-
-  opts.on("--debug") do
-    options["debug"] = true
-  end
-end
 
+parser = Shiba::Configure.make_options_parser(options)
+parser.banner = "Run a list of queries through shiba's analyzer."
 parser.parse!
 
 ["database", "username"].each do |opt|
@@ -67,12 +26,12 @@ end
 file = options.delete("file")
 file = File.open(file, "r") if file
 
-output = options.delete("output")
+output = options.delete("explain")
 output = File.open(output, 'w') if output
 
 Shiba.configure(options)
 
-schema_stats_fname = options["stats_file"]
+schema_stats_fname = options["stats"]
 
 if schema_stats_fname && !File.exist?(schema_stats_fname)
   $stderr.puts "No such file: #{schema_stats_fname}"
@@ -102,4 +61,4 @@ file = $stdin if file.nil?
 output = $stdout if output.nil?
 
 queries = Shiba::Analyzer.analyze(file, output, schema_stats, options)
-Shiba::Output.new(queries).make_web!
+Shiba::Output.new(queries, { 'output' => options['output'] }).make_web!

data/bin/redmine/sample_redmine.rb

@@ -45,7 +45,15 @@ end
 
 class Watcher < Redmine; end
 class Relation < Redmine; end
-class User < Redmine; end
+class User < Redmine
+  def enhance(hash, owner)
+    if rand < 0.01
+      hash['type'] = 'admin'
+    else
+      hash['type'] = 'user'
+    end
+  end
+end
 
 class Sampler
   def initialize(interested_in)
@@ -105,9 +113,17 @@ class Sampler
   def sample_model(model, params = {})
     # go exponentially up from 1.
     last = find_max(model, 0, 1, params)
+
     # go up from the last hit value.
-    if last
-      find_max(model, last.id, 1, params)
+    last = find_max(model, last.id, 1, params)
+
+    max_id = last.id
+    num_to_sample = (max_id * 0.05).to_i
+
+    num_to_sample.times do
+      instance = get_instance(model, rand(max_id), params)
+      extract_hash(instance) if instance
+      sleep(1)
     end
   end
 

data/lib/shiba.rb

@@ -1,11 +1,22 @@
 require "shiba/version"
+require "shiba/configure"
 require "mysql2"
 
 module Shiba
   class Error < StandardError; end
 
-  def self.configure(connection_hash)
-    @connection_hash = connection_hash
+  def self.configure(options)
+    @connection_hash = options.select { |k, v| ['username', 'database', 'host', 'password'].include?(k) }
+    @main_config = Configure.read_config_file(options['config'], "config/shiba.yml")
+    @index_config = Configure.read_config_file(options['index'], "config/shiba_index.yml")
+  end
+
+  def self.config
+    @main_config
+  end
+
+  def self.index_config
+    @index_config
   end
 
   def self.connection
@@ -18,4 +29,4 @@ module Shiba
 end
 
 # This goes at the end so that Shiba.root is defined.
-require "shiba/railtie" if defined?(Rails)
+require "shiba/railtie" if defined?(Rails)

data/lib/shiba/configure.rb

@@ -1,31 +1,103 @@
 require 'pathname'
-
+require 'pp'
 module Shiba
-    module Configure
-
-        # avoiding Rails dependency on the cli tools for now.
-        # yanked from https://github.com/rails/rails/blob/v5.0.5/railties/lib/rails/application/configuration.rb
-        def self.activerecord_configuration
-            yaml = Pathname.new("config/database.yml")
-
-            config = if yaml && yaml.exist?
-            require "yaml"
-            require "erb"
-            YAML.load(ERB.new(yaml.read).result) || {}
-            elsif ENV['DATABASE_URL']
-            # Value from ENV['DATABASE_URL'] is set to default database connection
-            # by Active Record.
-            {}
-            end
-
-            config
-        rescue Psych::SyntaxError => e
-            raise "YAML syntax error occurred while parsing #{yaml.to_s}. " \
-                "Please note that YAML must be consistently indented using spaces. Tabs are not allowed. " \
-                "Error: #{e.message}"
-        rescue => e
-            raise e, "Cannot load `#{path}`:\n#{e.message}", e.backtrace
+  module Configure
+
+    # avoiding Rails dependency on the cli tools for now.
+    # yanked from https://github.com/rails/rails/blob/v5.0.5/railties/lib/rails/application/configuration.rb
+    def self.activerecord_configuration
+      yaml = Pathname.new("config/database.yml")
+
+      config = if yaml && yaml.exist?
+                 require "yaml"
+                 require "erb"
+                 YAML.load(ERB.new(yaml.read).result) || {}
+               elsif ENV['DATABASE_URL']
+                 # Value from ENV['DATABASE_URL'] is set to default database connection
+                 # by Active Record.
+                 {}
+               end
+
+      config
+    rescue Psych::SyntaxError => e
+      raise "YAML syntax error occurred while parsing #{yaml.to_s}. " \
+        "Please note that YAML must be consistently indented using spaces. Tabs are not allowed. " \
+        "Error: #{e.message}"
+    rescue => e
+      raise e, "Cannot load `#{path}`:\n#{e.message}", e.backtrace
+    end
+
+    def self.read_config_file(option_file, default)
+      file_to_read = nil
+      if option_file
+        if !File.exist?(option_file)
+          $stderr.puts "no such file: #{option_file}"
+          exit 1
+        end
+        file_to_read = option_file
+      elsif File.exist?(default)
+        file_to_read = default
+      end
+
+      if file_to_read
+        YAML.load_file(file_to_read)
+      else
+        {}
+      end
+    end
+
+    def self.make_options_parser(options)
+      parser = OptionParser.new do |opts|
+        # note that the key to the hash needs to stay the same as the
+        # option name since we re-pass them
+        opts.on("-h","--host HOST", "sql host") do |h|
+          options["host"] = h
+        end
+
+        opts.on("-d","--database DATABASE", "sql database") do |d|
+          options["database"] = d
+        end
+
+        opts.on("-u","--username USER", "sql user") do |u|
+          options["username"] = u
+        end
+
+        opts.on("-p","--password PASSWORD", "sql password") do |p|
+          options["password"] = p
         end
 
+        opts.on("-c","--config FILE", "location of shiba.yml") do |f|
+          options["config"] = f
+        end
+
+        opts.on("-i","--index INDEX", "location of shiba_index.yml") do |i|
+          options["index"] = i.to_i
+        end
+
+        opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
+          options["limit"] = l.to_i
+        end
+
+        opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
+          options["stats"] = f
+        end
+
+        opts.on("-f", "--file FILE", "location of file containing queries") do |f|
+          options["file"] = f
+        end
+
+        opts.on("-e", "--explain FILE", "write explain JSON to file. default: stdout") do |f|
+          options["explain"] = f
+        end
+
+        opts.on("-o", "--output PATH", "path to put generated report in.  default: /tmp") do |p|
+          options["output"] = p
+        end
+
+        opts.on("-t", "--test", "analyze queries at --file instead of analyzing a process") do |f|
+          options["test"] = true
+        end
+      end
     end
-end
+  end
+end

data/lib/shiba/explain.rb

@@ -3,10 +3,10 @@ require 'shiba/index'
 
 module Shiba
   class Explain
-    def initialize(sql, stats, options = {})
+    def initialize(sql, stats, backtrace, options = {})
       @sql = sql
+      @backtrace = backtrace
 
-      @sql, _, @backtrace = @sql.partition(" /*shiba")
       if options[:force_key]
          @sql = @sql.sub(/(FROM\s*\S+)/i, '\1' + " FORCE INDEX(`#{options[:force_key]}`)")
       end
@@ -20,8 +20,6 @@ module Shiba
     end
 
     def as_json
-      @backtrace.chomp!("*/")
-
       {
         sql: @sql,
         table: get_table,
@@ -30,7 +28,7 @@ module Shiba
         cost: @cost,
         used_key_parts: first['used_key_parts'],
         possible_keys: first['possible_keys'],
-        backtrace: JSON.parse(@backtrace)
+        backtrace: @backtrace
       }
     end
 
@@ -217,7 +215,7 @@ module Shiba
     end
 
     def estimate_row_count_with_key(key)
-      Explain.new(@sql, @stats, force_key: key).estimate_row_count
+      Explain.new(@sql, @stats, @backtrace, force_key: key).estimate_row_count
     rescue Mysql2::Error => e
       if /Key .+? doesn't exist in table/ =~ e.message
         return nil
@@ -226,7 +224,32 @@ module Shiba
       raise e
     end
 
+    def ignore?
+      !!ignore_line_and_backtrace_line
+    end
+
+    def ignore_line_and_backtrace_line
+      ignore_files = Shiba.config['ignore']
+      if ignore_files
+        ignore_files.each do |i|
+          file, method = i.split('#')
+          @backtrace.each do |b|
+            next unless b.include?(file)
+            next if method && !b.include?(method)
+            return [i, b]
+          end
+        end
+      end
+      nil
+    end
+
     def run_checks!
+      if ignore?
+        @cost = 0
+        messages << "ignored"
+        return
+      end
+
       @cost = estimate_row_count
     end
   end

data/lib/shiba/output.rb

@@ -5,28 +5,26 @@ require 'erb'
 
 module Shiba
   class Output
-
     OUTPUT_PATH = "/tmp/shiba_results"
-    JS_PATH = OUTPUT_PATH + "/js"
 
     WEB_PATH = File.dirname(__FILE__) + "/../../web"
     def self.tags
       @tags ||= YAML.load_file(File.dirname(__FILE__) + "/output/tags.yaml")
     end
 
-    def self.from_file(fname)
-      queries = []
-      File.open(fname, "r") do |f|
-        while line = f.gets
-          queries << JSON.parse(line)
-        end
-      end
-      new(queries)
+    def initialize(queries, options = {})
+      @queries = queries
+      @options = options
     end
 
+    def output_path
+      path ||= File.join(@options['output'], "shiba_results") if @options['output']
+      path ||= Dir.pwd + "/log/shiba_results" if File.exist?(Dir.pwd + "/log")
+      path ||= OUTPUT_PATH
+    end
 
-    def initialize(queries)
-      @queries = queries
+    def js_path
+      File.join(output_path, "js")
     end
 
     def remote_url
@@ -40,11 +38,11 @@ module Shiba
     end
 
     def make_web!
-      FileUtils.mkdir_p(JS_PATH)
+      FileUtils.mkdir_p(js_path)
 
       js = Dir.glob(WEB_PATH + "/dist/*.js").map { |f| File.basename(f) }
       js.each do |f|
-        system("cp #{WEB_PATH}/dist/#{f} #{JS_PATH}")
+        system("cp #{WEB_PATH}/dist/#{f} #{js_path}")
       end
 
       data = {
@@ -54,14 +52,14 @@ module Shiba
         url: remote_url
       }
 
-      system("cp #{WEB_PATH}/*.css #{OUTPUT_PATH}")
+      system("cp #{WEB_PATH}/*.css #{output_path}")
 
       erb = ERB.new(File.read(WEB_PATH + "/../web/results.html.erb"))
-      File.open(OUTPUT_PATH + "/results.html", "w+") do |f|
+      File.open(output_path + "/results.html", "w+") do |f|
         f.write(erb.result(binding))
       end
 
-      puts "done, results are in " + "/tmp/shiba_results/results.html"
+      puts "done, results are in " + File.join(output_path, "results.html")
     end
   end
 end

data/lib/shiba/output/tags.yaml

@@ -38,7 +38,12 @@ access_type_tablescan:
 fuzzed_data:
   title: Guessed Table Size
   description: |
-    There wasn't enough data in your local environment to adequately measure this query, so we
-    pretended this table had a bunch of data in it. To improve results, please dump your
-    production index statistics into Shiba.
+    We're not sure how much data this table will hold in the future, so we've pretended
+    there's 6000 rows in it.  This can lead to a lot of false positives. To
+    improve results, please give shiba your index statistics.
+  level: info
+ignored:
+  title: Ignored Query
+  description: |
+    This query matched an "ignore" rule in shiba.yml, so we skipped any further analysis on it.
   level: info

data/lib/shiba/query.rb

@@ -15,7 +15,15 @@ module Shiba
     end
 
     def initialize(sql, stats)
-      @sql = sql
+      @sql, _, @backtrace = sql.partition(" /*shiba")
+
+      if @backtrace.length > 0
+        @backtrace.chomp!("*/")
+        @backtrace = JSON.parse(@backtrace)
+      else
+        @backtrace = []
+      end
+
       @stats = stats
       @@index += 1
       @index = @@index
@@ -23,12 +31,17 @@ module Shiba
 
     attr_reader :sql, :index
 
+
     def fingerprint
       @fingerprint ||= self.class.get_fingerprint(@sql)
     end
 
     def explain
-      Explain.new(@sql, @stats)
+      Explain.new(@sql, @stats, @backtrace)
+    end
+
+    def backtrace
+      @backtrace
     end
   end
 end

data/lib/shiba/query_watcher.rb

@@ -4,13 +4,14 @@ require 'rails'
 
 module Shiba
   class QueryWatcher
-    FINGERPRINTS = {}
     IGNORE = /\.rvm|gem|vendor\/|rbenv|seed|db|shiba|test|spec/
 
     def self.watch(file)
       new(file).watch
     end
 
+    attr_reader :queries
+
     def initialize(file)
       @file = file
       # fixme mem growth on this is kinda nasty
@@ -23,11 +24,13 @@ module Shiba
         sql = payload[:sql]
 
         if sql.start_with?("SELECT")
-          lines = app_backtrace
-          if lines && !@queries[sql]
-            @file.puts("#{sql} /*shiba#{lines}*/" )
+          fingerprint = Query.get_fingerprint(sql)
+          if !@queries[fingerprint]
+            if lines = app_backtrace
+              @file.puts("#{sql} /*shiba#{lines}*/")
+            end
           end
-          @queries[sql] = true
+          @queries[fingerprint] = true
         end
       end
     end

data/lib/shiba/version.rb

@@ -1,3 +1,3 @@
 module Shiba
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/web/results.html.erb

@@ -94,11 +94,13 @@
           <query v-for="query in queries" v-bind:query="query" v-bind:key="query.sql" v-bind:tags="tags"></query>
         </div>
         <div class="row">
-          <div class="col-12">We also found {{ queriesLow.length }} queries that look fine.</div>
+          <div class="col-12">We also found <a href="#" v-on:click.prevent="lowExpanded = !lowExpanded">{{ queriesLow.length }} queries</a> that look fine.</div>
         </div>
-        <div class="queries">
+        <a name="lowExapnded"></a>
+        <div class="queries" v-if="lowExpanded">
           <query v-for="query in queriesLow" v-bind:query="query" v-bind:key="query.sql" v-bind:tags="tags"></query>
         </div>
+        <div style="height:50px"></div>
       </div>
     </div>
 
@@ -188,7 +190,8 @@
       data: {
         queries: queriesByTable,
         queriesLow: queriesByTableLow,
-        tags: data.tags
+        tags: data.tags,
+        lowExpanded: false
       },
       methods: { }
     });

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shiba
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Ben Osheroff