shiba 0.1.2 → 0.2.0

data/lib/shiba.rb

@@ -1,12 +1,14 @@
 require "shiba/version"
 require "shiba/configure"
 require "mysql2"
+require "pp"
+require "byebug" if ENV['SHIBA_DEBUG']
 
 module Shiba
   class Error < StandardError; end
 
   def self.configure(options)
-    @connection_hash = options.select { |k, v| ['username', 'database', 'host', 'password'].include?(k) }
+    @connection_hash = options.select { |k, v| [ 'default_file', 'default_group', 'username', 'database', 'host', 'password'].include?(k) }
     @main_config = Configure.read_config_file(options['config'], "config/shiba.yml")
     @index_config = Configure.read_config_file(options['index'], "config/shiba_index.yml")
   end

data/lib/shiba/analyzer.rb

@@ -69,7 +69,7 @@ module Shiba
     protected
 
     def dump_error(e, query)
-      $stderr.puts "got exception trying to explain: #{e.message}"
+      $stderr.puts "got #{e.class.name} exception trying to explain: #{e.message}"
       $stderr.puts "query: #{query.sql} (index #{query.index})"
       $stderr.puts e.backtrace.join("\n")
     end
@@ -79,10 +79,7 @@ module Shiba
       begin
         explain = query.explain
       rescue Mysql2::Error => e
-        # we're picking up crap on the command-line that's not good SQL.  ignore it.
-        if !(e.message =~ /You have an error in your SQL syntax/)
-          dump_error(e, query)
-        end
+        dump_error(e, query) if verbose?
       rescue StandardError => e
         dump_error(e, query)
       end
@@ -96,5 +93,9 @@ module Shiba
     def write(line)
       @output.puts(line)
     end
+
+    def verbose?
+      @options['verbose'] == true
+    end
   end
 end

data/lib/shiba/backtrace.rb

@@ -0,0 +1,56 @@
+require 'open3'
+
+module Shiba
+  module Backtrace
+    IGNORE = /\.rvm|gem|vendor\/|rbenv|seed|db|shiba|test|spec/
+
+    # 8 backtrace lines starting from the app caller, cleaned of app/project cruft.
+    def self.from_app
+      app_line_idx = caller_locations.index { |line| line.to_s !~ IGNORE }
+      if app_line_idx == nil
+        return
+      end
+
+      caller_locations(app_line_idx+1, 8).map do |loc|
+        clean!(loc.to_s)
+      end
+    end
+
+    def self.clean!(line)
+      line.sub!(backtrace_ignore_pattern, '')
+      line
+    end
+
+    protected
+
+    def self.backtrace_ignore_pattern
+      @roots ||= begin
+        paths = Gem.path
+        paths << Rails.root.to_s if defined?(Rails.root)
+        paths << repo_root
+        paths << ENV['HOME']
+        paths.uniq!
+        paths.compact!
+        # match and replace longest path first
+        paths.sort_by!(&:size).reverse!
+
+        r = Regexp.new(paths.map {|r| Regexp.escape(r) }.join("|"))
+        # kill leading slash
+        /(#{r})\/?/
+      end
+    end
+
+    # /user/git_repo => "/user/git_repo"
+    # /user/not_a_repo => nil
+    def self.repo_root
+      root = nil
+      Open3.popen3('git rev-parse --show-toplevel') {|_,o,_,_|
+        if root = o.gets
+          root = root.chomp
+        end
+      }
+      root
+    end
+
+  end
+end

data/lib/shiba/checker.rb

@@ -0,0 +1,103 @@
+require 'json'
+require 'open3'
+
+require 'shiba/diff'
+require 'shiba/backtrace'
+
+module Shiba
+  class Checker
+    Result = Struct.new(:status, :message, :problems)
+
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+    end
+
+    def run(log)
+      msg = nil
+
+      if options['verbose']
+        puts cmd
+      end
+
+      if changes.empty?
+        if options['verbose']
+          msg = "No changes found in git"
+        end
+        return Result.new(:pass, msg)
+      end
+
+      explains = select_lines_with_changed_files(log)
+      problems = explains.select { |explain| explain["cost"] && explain["cost"] > MAGIC_COST }
+
+      problems.select! do |problem|
+        backtrace_has_updated_line?(problem["backtrace"], updated_lines)
+      end
+
+      if problems.empty?
+        if options['verbose']
+          msg = "No problems found"
+        end
+
+        return Result.new(:pass, msg)
+      end
+
+      return Result.new(:fail, "Potential problems", problems)
+    end
+
+    protected
+
+    def backtrace_has_updated_line?(backtrace, updates)
+      backtrace.any? do |bl|
+        updates.any? do |path, lines|
+          next if !bl.start_with?(path)
+          bl =~ /:(\d+):/
+          lines.include?($1.to_i)
+        end
+      end
+    end
+
+    def select_lines_with_changed_files(log)
+      patterns = changes.split("\n").map { |path| "-e #{path}" }.join(" ")
+      json_lines = `grep #{log} #{patterns}`
+      json_lines.each_line.map { |line| JSON.parse(line) }
+    end
+
+    def changes
+      @changes ||= begin
+        result = `git diff#{cmd} --name-only --diff-filter=d`
+        if $?.exitstatus != 0
+          error("Failed to read changes", $?.exitstatus)
+        end
+
+        result
+      end
+    end
+
+    def updated_lines
+      return @updated_lines if @updated_lines
+
+      Open3.popen3("git diff#{cmd} --unified=0 --diff-filter=d") {|_,o,_,_|
+        @updated_lines = Shiba::Diff.new(o).updated_lines
+      }
+
+      @updated_lines.map! do |path, lines|
+        [ Shiba::Backtrace.clean!(path), lines ]
+      end
+    end
+
+    def cmd
+      cmd = case
+      when options["staged"]
+        " --staged"
+      when options["unstaged"]
+        ""
+      else
+        commit = " HEAD"
+        commit << "...#{options["branch"]}" if options["branch"]
+        commit
+      end
+    end
+  end
+end

data/lib/shiba/configure.rb

@@ -27,6 +27,13 @@ module Shiba
       raise e, "Cannot load `#{path}`:\n#{e.message}", e.backtrace
     end
 
+    # loosely based on https://dev.mysql.com/doc/refman/8.0/en/option-files.html
+    def self.mysql_config_path
+      paths = [ File.join(Dir.home, '.mylogin.cnf'), File.join(Dir.home, '.my.cnf')  ]
+
+      paths.detect { |p| File.exist?(p) }
+    end
+
     def self.read_config_file(option_file, default)
       file_to_read = nil
       if option_file
@@ -78,25 +85,38 @@ module Shiba
           options["limit"] = l.to_i
         end
 
-        opts.on("-s","--stats FILES", "location of index statistics tsv file") do |f|
-          options["stats"] = f
-        end
-
         opts.on("-f", "--file FILE", "location of file containing queries") do |f|
           options["file"] = f
         end
 
-        opts.on("-e", "--explain FILE", "write explain JSON to file. default: stdout") do |f|
-          options["explain"] = f
+        opts.on("-j", "--json [FILE]", "write JSON report here. default: to stdout") do |f|
+          if f
+            options["json"] = File.open(f, 'w')
+          else
+            options["json"] = $stdout
+          end
         end
 
-        opts.on("-o", "--output PATH", "path to put generated report in.  default: /tmp") do |p|
-          options["output"] = p
+        opts.on("-h", "--html FILE", "write html report here. Default to /tmp/explain.html") do |h|
+          options["html"] = h
         end
 
         opts.on("-t", "--test", "analyze queries at --file instead of analyzing a process") do |f|
           options["test"] = true
         end
+
+        opts.on("-v", "--verbose", "print internal runtime information") do
+           options["verbose"] = true
+        end
+
+        # This naming seems to be mysql convention, maybe we should just do our own thing though.
+        opts.on("--login-path", "The option group from the mysql config file to read from") do |f|
+          options["default_group"] = f
+        end
+
+        opts.on("--default-extras-file", "The option file to read mysql configuration from") do |f|
+          options["default_file"] = f
+        end
       end
     end
   end

data/lib/shiba/diff.rb

@@ -0,0 +1,119 @@
+module Shiba
+  class Diff
+    # +++ b/config/environments/test.rb
+    FILE_PATTERN = /\A\+\+\+ b\/(.*?)\Z/
+
+    # @@ -177,0 +178 @@ ...
+    # @@ -177,0 +178,5 @@ ...
+    # @@ -21 +24 @@ ...
+    LINE_PATTERN = /\A@@ \-\d+,?\d+? \+(\d+),?(\d+)? @@/
+
+    # via https://developer.github.com/v3/pulls/comments/#create-a-comment
+    # The position value equals the number of lines down from the first "@@" hunk header
+    # in the file you want to add a comment.
+
+    # diff = `git diff --unified=0`
+    # parse_diff(StringIO.new(diff))
+    # => "hello.rb:1"
+    # => "hello.rb:2"
+    # => "test.rb:5"
+
+  # For simplicity, the default output of git diff is not supported.
+  # The expected format is from 'git diff unified=0'
+
+  attr_reader :status
+
+  def initialize(file)
+    @diff = file
+    @status = :new
+  end
+
+  # Returns the file and line numbers that contain inserts. Deletions are ignored.
+  def updated_lines
+    io = @diff.each_line
+    path = nil
+
+    found = []
+
+    while true
+      line = io.next
+      if line =~ FILE_PATTERN
+        path = $1
+      end
+
+      if hunk_header?(line)
+        line_numbers = line_numbers_for_destination(line)
+        found << [ path, line_numbers ]
+      end
+    end
+  rescue StopIteration
+    return found
+  end
+
+  # Returns the position in the diff, after the relevant file header,
+  # that contains the specified file/lineno modification.
+  # Only supports finding the position in the destination / newest version of the file.
+  def find_position(path, line_number)
+    io = @diff.each_line # maybe redundant?
+
+    file_header = "+++ b/#{path}\n" # fixme
+    if !io.find_index(file_header)
+      @status = :file_not_found
+      return
+    end
+
+    line = io.peek
+    if !hunk_header?(line)
+      raise StandardError.new("Expected hunk header to be after file header, but got '#{line}'")
+    end
+
+    pos = 0
+
+    while true
+      line = io.next
+      pos += 1
+
+      if file_header?(line)
+        @status = :line_not_found
+        return
+      end
+
+      if !hunk_header?(line)
+        next
+      end
+
+      line_numbers = line_numbers_for_destination(line)
+
+      if destination_position = line_numbers.find_index(line_number)
+        @status = :found_position
+        return pos + find_hunk_index(io, destination_position)
+      end
+    end
+    rescue StopIteration
+      @status = :line_not_found
+    end
+
+    protected
+
+    def find_hunk_index(hunk, pos)
+      line, idx = hunk.with_index.select { |l,idx| !l.start_with?('-') }.take(pos+1).last
+      idx
+    end
+
+    def file_header?(line)
+      line.match?(FILE_PATTERN)
+    end
+
+    def hunk_header?(line)
+      line.match?(LINE_PATTERN)
+    end
+
+    def line_numbers_for_destination(diff_line)
+      diff_line =~ LINE_PATTERN
+      line = $1.to_i
+      line_count = ($2 && $2.to_i) || 0
+      line..line+line_count
+    end
+
+  end
+end

data/lib/shiba/explain.rb

@@ -13,8 +13,8 @@ module Shiba
 
       @options = options
       ex = Shiba.connection.query("EXPLAIN FORMAT=JSON #{@sql}").to_a
-      json = JSON.parse(ex.first['EXPLAIN'])
-      @rows = self.class.transform_json(json['query_block'])
+      @explain_json = JSON.parse(ex.first['EXPLAIN'])
+      @rows = self.class.transform_json(@explain_json['query_block'])
       @stats = stats
       run_checks!
     end
@@ -23,11 +23,15 @@ module Shiba
       {
         sql: @sql,
         table: get_table,
+        table_size: table_size,
         key: first_key,
         tags: messages,
         cost: @cost,
+        return_size: @return_size,
+        severity: severity,
         used_key_parts: first['used_key_parts'],
         possible_keys: first['possible_keys'],
+        raw_explain: humanized_explain,
         backtrace: @backtrace
       }
     end
@@ -43,7 +47,7 @@ module Shiba
       table
     end
 
-    def self.transform_table(table)
+    def self.transform_table(table, extra = {})
       t = table
       res = {}
       res['table'] = t['table_name']
@@ -57,24 +61,30 @@ module Shiba
         res['possible_keys'] = t['possible_keys']
       end
       res['using_index'] = t['using_index'] if t['using_index']
+
+      res.merge!(extra)
+
       res
     end
 
-    def self.transform_json(json, res = [])
+    def self.transform_json(json, res = [], extra = {})
       rows = []
 
-      if json['ordering_operation']
-        return transform_json(json['ordering_operation'])
+      if (ordering = json['ordering_operation'])
+        index_walk = (ordering['using_filesort'] == false)
+        return transform_json(json['ordering_operation'], res, { "index_walk" => index_walk } )
       elsif json['duplicates_removal']
-        return transform_json(json['duplicates_removal'])
+        return transform_json(json['duplicates_removal'], res, extra)
+      elsif json['grouping_operation']
+        return transform_json(json['grouping_operation'], res, extra)
       elsif !json['nested_loop'] && !json['table']
         return [{'Extra' => json['message']}]
       elsif json['nested_loop']
         json['nested_loop'].map do |nested|
-          transform_json(nested, res)
+          transform_json(nested, res, extra)
         end
       elsif json['table']
-        res << transform_table(json['table'])
+        res << transform_table(json['table'], extra)
       end
       res
     end
@@ -112,15 +122,12 @@ module Shiba
       first.merge(cost: cost, messages: messages)
     end
 
-    IGNORE_PATTERNS = [
-      /No tables used/,
-      /Impossible WHERE/,
-      /Select tables optimized away/,
-      /No matching min\/max row/
-    ]
-
     def table_size
-      Shiba::Index.count(first["table"], @stats)
+      @stats.table_count(first['table'])
+    end
+
+    def fuzzed?(table)
+      @stats.fuzzed?(first['table'])
     end
 
     def no_matching_row_in_const_table?
@@ -128,7 +135,6 @@ module Shiba
     end
 
     def ignore_explain?
-      first_extra && IGNORE_PATTERNS.any? { |p| first_extra =~ p }
     end
 
     def derived?
@@ -137,7 +143,18 @@ module Shiba
 
     # TODO: need to parse SQL here I think
     def simple_table_scan?
-      @rows.size == 1 && (@sql !~ /where/i || @sql =~ /where\s*1=1/i) && (@sql !~ /order by/i)
+      @rows.size == 1 && first['using_index'] && (@sql !~ /order by/i)
+    end
+
+    def severity
+      case @cost
+      when 0..100
+        "low"
+      when 100..1000
+        "medium"
+      when 1000..1_000_000_000
+        "high"
+      end
     end
 
     def limit
@@ -148,53 +165,115 @@ module Shiba
       end
     end
 
-    def tag_query_type
-      access_type = first['access_type']
+    def aggregation?
+      @sql =~ /select\s*(.*?)from/i
+      select_fields = $1
+      select_fields =~ /min|max|avg|count|sum|group_concat\s*\(.*?\)/i
+    end
 
-      return unless access_type
-      access_type = 'tablescan' if access_type == 'ALL'
-      messages << "access_type_" + access_type
+    def self.check(c)
+      @checks ||= []
+      @checks << c
+    end
+
+    def self.get_checks
+      @checks
+    end
+
+    check :check_query_is_ignored
+    def check_query_is_ignored
+      if ignore?
+        messages << "ignored"
+        @cost = 0
+      end
     end
 
-    def estimate_row_count
+    check :check_no_matching_row_in_const_table
+    def check_no_matching_row_in_const_table
       if no_matching_row_in_const_table?
         messages << "access_type_const"
         first['key'] = 'PRIMARY'
-        return 0
+        @cost = 1
       end
+    end
 
-      return 0 if ignore_explain?
+    IGNORE_PATTERNS = [
+      /No tables used/,
+      /Impossible WHERE/,
+      /Select tables optimized away/,
+      /No matching min\/max row/
+    ]
 
-      messages << "fuzzed_data" if Shiba::Index.fuzzed?(first_table, @stats)
+    check :check_query_shortcircuits
+    def check_query_shortcircuits
+      if first_extra && IGNORE_PATTERNS.any? { |p| first_extra =~ p }
+        @cost = 0
+      end
+    end
+
+    check :check_fuzzed
+    def check_fuzzed
+      messages << "fuzzed_data" if fuzzed?(first_table)
+    end
 
+    check :check_simple_table_scan
+    def check_simple_table_scan
       if simple_table_scan?
         if limit
           messages << 'limited_tablescan'
+          @cost = limit
         else
-          messages << 'access_type_tablescan'
+          tag_query_type
+          @cost = @stats.estimate_key(first_table, first_key, first['used_key_parts'])
         end
-
-        return limit || table_size
       end
+    end
 
+    check :check_derived
+    def check_derived
       if derived?
         # select count(*) from ( select 1 from foo where blah )
         @rows.shift
-        return estimate_row_count
+        return run_checks!
+      end
+    end
+
+
+    check :tag_query_type
+    def tag_query_type
+      access_type = first['access_type']
+
+      if access_type.nil?
+        @cost = 0
+        return
       end
 
-      tag_query_type
+      access_type = 'tablescan' if access_type == 'ALL'
+      messages << "access_type_" + access_type
+    end
 
+    #check :check_index_walk
+    # disabling this one for now, it's not quite good enough and has a high
+    # false-negative rate.
+    def check_index_walk
+      if first['index_walk']
+        @cost = limit
+        messages << 'index_walk'
+      end
+    end
+
+    check :check_key_size
+    def check_key_size
       # TODO: if possible_keys but mysql chooses NULL, this could be a test-data issue,
       # pick the best key from the list of possibilities.
       #
       if first_key
-        Shiba::Index.estimate_key(first_table, first_key, first['used_key_parts'], @stats)
+        @cost = @stats.estimate_key(first_table, first_key, first['used_key_parts'])
       else
         if first['possible_keys'].nil?
           # if no possibile we're table scanning, use PRIMARY to indicate that cost.
           # note that this can be wildly inaccurate bcs of WHERE + LIMIT stuff.
-          Shiba::Index.count(first_table, @stats)
+          @cost = table_size
         else
           if @options[:force_key]
             # we were asked to force a key, but mysql still told us to fuck ourselves.
@@ -202,20 +281,37 @@ module Shiba
             #
             # there seems to be cases where mysql lists `possible_key` values
             # that it then cannot use, seen this in OR queries.
-            return Shiba::Index.count(first_table, @stats)
+            @cost = table_size
+          else
+            possibilities = [table_size]
+            possibilities += first['possible_keys'].map do |key|
+              estimate_row_count_with_key(key)
+            end
+            @cost = possibilities.compact.min
           end
-
-          possibilities = [Shiba::Index.count(first_table, @stats)]
-          possibilities += first['possible_keys'].map do |key|
-            estimate_row_count_with_key(key)
-          end
-          possibilities.compact.min
         end
       end
     end
 
+    def check_return_size
+      if limit
+        @return_size = limit
+      elsif aggregation?
+        @return_size = 1
+      else
+        @return_size = @cost
+      end
+
+      if @return_size && @return_size > 100
+        messages << "retsize_bad"
+      else
+        messages << "retsize_good"
+      end
+    end
+
     def estimate_row_count_with_key(key)
-      Explain.new(@sql, @stats, @backtrace, force_key: key).estimate_row_count
+      explain = Explain.new(@sql, @stats, @backtrace, force_key: key)
+      explain.run_checks!
     rescue Mysql2::Error => e
       if /Key .+? doesn't exist in table/ =~ e.message
         return nil
@@ -244,14 +340,18 @@ module Shiba
     end
 
     def run_checks!
-      if ignore?
-        @cost = 0
-        messages << "ignored"
-        return
+      self.class.get_checks.each do |check|
+        res = send(check)
+        break if @cost
       end
+      check_return_size
+      @cost
+    end
 
-      @cost = estimate_row_count
+    def humanized_explain
+      h = @explain_json['query_block'].dup
+      %w(select_id cost_info).each { |i| h.delete(i) }
+      h
     end
   end
 end
-