shiba 0.5.0 → 0.6.0

data/lib/shiba/activerecord_integration.rb

@@ -22,6 +22,26 @@ module Shiba
       @installed = true
     end
 
+    def self.connection_options
+        cx = ActiveRecord::Base.connection.raw_connection
+        if cx.respond_to?(:query_options)
+          # mysql
+          c = cx.query_options.merge(server: 'mysql')
+        else
+          # postgres
+          c = { host: cx.host, database: cx.db, username: cx.user, password: cx.pass, port: cx.port, server: 'postgres' }
+        end
+
+        options = {
+          'host' =>     c[:host],
+          'database' => c[:database],
+          'user' =>     c[:username],
+          'password' => c[:password],
+          'port' =>     c[:port],
+          'server' =>   c[:server],
+        }
+    end
+
     protected
 
     def self.start_watcher
@@ -64,26 +84,8 @@ module Shiba
     end
 
     def self.database_args
-      cx = ActiveRecord::Base.connection.raw_connection
-      if cx.respond_to?(:query_options)
-        # mysql
-        c = cx.query_options.merge(server: 'mysql')
-      else
-        # postgres
-        c = { host: cx.host, database: cx.db, username: cx.user, password: cx.pass, port: cx.port, server: 'postgres' }
-      end
-
-      options = {
-        'host':     c[:host],
-        'database': c[:database],
-        'user':     c[:username],
-        'password': c[:password],
-        'port':     c[:port],
-        'server':   c[:server]
-      }
-
       # port can be a Fixnum
-      options.reject { |k,v| v.nil? || v.respond_to?(:empty?) && v.empty? }.map { |k,v| "--#{k} #{v}" }.join(" ")
+      connection_options.reject { |k,v| v.nil? || v.respond_to?(:empty?) && v.empty? }.map { |k,v| "--#{k} #{v}" }.join(" ")
     end
 
   end

data/lib/shiba/analyzer.rb

@@ -81,7 +81,7 @@ module Shiba
       end
       return nil unless explain
 
-      if explain.other_paths.any?
+      if explain.severity != 'none' && explain.other_paths.any?
         paths = [explain] + explain.other_paths
         explain = paths.sort { |a, b| a.cost - b.cost }.first
       end

data/lib/shiba/configure.rb

@@ -7,19 +7,25 @@ module Shiba
 
     # avoiding Rails dependency on the cli tools for now.
     # yanked from https://github.com/rails/rails/blob/v5.0.5/railties/lib/rails/application/configuration.rb
-    def self.activerecord_configuration
-      yaml = Pathname.new("config/database.yml")
+    def self.activerecord_configuration(config_path = "config/database.yml")
+      yaml = Pathname.new(config_path)
 
       config = if yaml && yaml.exist?
                  require "yaml"
                  require "erb"
                  YAML.load(ERB.new(yaml.read).result) || {}
-               elsif ENV['DATABASE_URL']
-                 # Value from ENV['DATABASE_URL'] is set to default database connection
-                 # by Active Record.
-                 {}
                end
 
+      env     = ENV["RAILS_ENV"] || "test"
+      config  = config[env]
+      adapter = config.delete("adapter")
+
+      if adapter == "mysql2"
+        config["server"] = "mysql"
+      else
+        config["server"] = adapter
+      end
+
       config
     rescue Psych::SyntaxError => e
       raise "YAML syntax error occurred while parsing #{yaml.to_s}. " \

data/lib/shiba/connection/mysql.rb

@@ -1,5 +1,6 @@
 require 'mysql2'
 require 'json'
+require 'shiba/parsers/mysql_select_fields'
 
 module Shiba
   class Connection
@@ -21,9 +22,69 @@ module Shiba
         @connection.query(sql)
       end
 
+      def tables
+        @connection.query("show tables").map { |r| r.values.first }
+      end
+
+      def each_column_size
+        tables.each do |t|
+          sql = <<-EOL
+            select * from information_schema.columns where table_schema = DATABASE()
+            and table_name = '#{t}'
+          EOL
+          columns = @connection.query(sql)
+          col_hash = Hash[columns.map { |c| [c['COLUMN_NAME'], c] }]
+          estimate_column_sizes(t, col_hash)
+
+          col_hash.each do |c, h|
+            yield(t, c, h['size'])
+          end
+        end
+      end
+
+      def estimate_column_sizes(table, hash)
+        columns_to_sample = []
+        hash.each do |name, row|
+          row['size'] = case row['DATA_TYPE']
+          when 'tinyint', 'year', 'enum', 'bit'
+            1
+          when 'smallint'
+            2
+          when 'mediumint', 'date', 'time'
+            3
+          when 'int', 'decimal', 'float', 'timestamp'
+            4
+          when 'bigint', 'datetime', 'double'
+            8
+          else
+            columns_to_sample << name
+            nil
+          end
+        end
+
+        return unless columns_to_sample.any?
+
+        select_fields = columns_to_sample.map do |c|
+          "AVG(LENGTH(`#{c}`)) as `#{c}`"
+        end.join(', ')
+
+        res = @connection.query("select #{select_fields}, count(*) as cnt from ( select * from `#{table}` limit 10000 ) as v").first
+        if res['cnt'] == 0
+          # muggles, no data. impossible to know actual size of blobs/varchars, safer to err on side of 0
+          res.keys.each do |c|
+            hash[c] && hash[c]['size'] = 0
+          end
+        else
+          res.each do |k, v|
+            hash[k] && hash[k]['size'] = v.to_i
+          end
+        end
+
+        hash
+      end
+
       def analyze!
-        @connection.query("show tables").each do |row|
-          t = row.values.first
+        tables.each do |t|
           @connection.query("analyze table `#{t}`") rescue nil
         end
       end
@@ -41,7 +102,15 @@ module Shiba
 
       def explain(sql)
         rows = query("EXPLAIN FORMAT=JSON #{sql}").to_a
-        JSON.parse(rows.first['EXPLAIN'])
+        explain = JSON.parse(rows.first['EXPLAIN'])
+        warnings = query("show warnings").to_a
+        [explain, parse_select_fields(warnings)]
+      end
+
+      def parse_select_fields(warnings)
+        normalized_sql = warnings.detect { |w| w["Code"] == 1003 }["Message"]
+
+        Parsers::MysqlSelectFields.new(normalized_sql).parse_fields
       end
 
       def mysql?

data/lib/shiba/connection/postgres.rb

@@ -81,7 +81,10 @@ module Shiba
 
       def explain(sql)
         rows = query("EXPLAIN (FORMAT JSON) #{sql}").to_a
-        rows.first["QUERY PLAN"]
+        [rows.first["QUERY PLAN"], {}]
+      end
+
+      def each_column_size
       end
 
       def mysql?

data/lib/shiba/console.rb

@@ -0,0 +1,165 @@
+require 'shiba'
+require 'shiba/activerecord_integration'
+require 'shiba/configure'
+require 'shiba/analyzer'
+require 'shiba/table_stats'
+require 'shiba/reviewer'
+
+module Shiba
+  # Provides a 'shiba' command to analyze queries from the console.
+  # If required in IRB or Pry, the shiba command will automatically be available,
+  # as it's injected into those consoles at the bottom of this file.
+  #
+  # Example:
+  # require 'shiba/console'
+  #
+  # shiba User.all
+  # => <shiba results>
+  # shiba "select * from users"
+  # => <shiba results>
+  module Console
+
+    def shiba(query)
+      @command ||= Command.new(self)
+      @command.execute(query)
+    end
+
+    class ExplainRecord
+
+      def initialize(fields)
+        @fields = fields
+      end
+
+      def comments
+        # renderer expects json / key strings
+        json = JSON.parse(JSON.dump(@fields))
+        renderer.render(json)
+      end
+
+      def md5
+        @fields[:md5]
+      end
+
+      def severity
+        @fields[:severity]
+      end
+
+      def sql
+        @fields[:sql]
+      end
+
+      def time
+        @fields[:cost]
+      end
+
+      def raw_explain
+        @fields[:raw_explain]
+      end
+
+      def message
+        msg = "\n"
+        msg << "Severity: #{severity}\n"
+        msg << "----------------------------\n"
+        msg << comments
+        msg << "\n\n"
+      end
+
+      def help
+        "Available methods: #{self.class.public_instance_methods(false)}"
+      end
+
+      def inspect
+        "#{to_s}: '#{sql}'. Call the 'help' method on this object for more info."
+      end
+
+      protected
+
+      def renderer
+        @renderer ||= Review::CommentRenderer.new(tags)
+      end
+
+      def tags
+        @tags ||= YAML.load_file(Shiba::TEMPLATE_FILE)
+      end
+
+    end
+
+    class Command
+
+      def initialize(context)
+        @context = context
+      end
+
+      def execute(query)
+        if !valid_query?(query)
+          msg = "Query does not appear to be a valid relation or select sql string"
+          msg << "\n#{usage}"
+          puts msg
+          return
+        end
+
+        result = explain(query)
+        if result == nil
+          puts "Unable to analyze query, please check the SQL syntax for typos."
+          return
+        end
+
+        record = ExplainRecord.new(result)
+        puts record.message
+        record
+      end
+
+      private
+
+      def usage
+        "Examples:
+        shiba User.all
+        shiba \"select * from users\""
+      end
+
+      def valid_query?(query)
+        query.respond_to?(:to_sql) ||
+          query.respond_to?(:=~) && query =~ /\Aselect/i
+      end
+
+      def explain(query)
+        query = query.to_sql if query.respond_to?(:to_sql)
+        Shiba.configure(connection_options)
+        analyzer = Shiba::Analyzer.new(nil, null, stats, { 'sql' => query })
+        result = analyzer.analyze.first
+      end
+
+      def connection_options
+        case
+        when defined?(ActiveRecord)
+          ActiveRecordIntegration.connection_options
+        when File.exist?("config/database.yml")
+          Shiba::Configure.activerecord_configuration
+        when File.exist?("test/database.yml.example")
+          Shiba::Configure.activerecord_configuration("test/database.yml.example")
+        else
+          raise Shiba::Error.new("ActiveRecord is currently required to analyze queries from the console.")
+        end
+      end
+
+      def stats
+        @stats ||= Shiba::TableStats.new(Shiba.index_config, Shiba.connection, {})
+      end
+
+      def null
+        @null ||= File.open(File::NULL, "w")
+      end
+
+      def puts(message)
+        out = @context.respond_to?(:puts) ? @context : $stdout
+        out.puts(message)
+      end
+
+    end
+
+  end
+end
+
+if defined?(Pry) || defined?(IRB)
+  TOPLEVEL_BINDING.eval('self').extend Shiba::Console
+end

data/lib/shiba/explain.rb

@@ -12,6 +12,8 @@ module Shiba
     COST_PER_ROW_SORT = 1.0e-07
     COST_PER_ROW_RETURNED = 3.0e-05
 
+    COST_PER_KB_RETURNED = 0.0004
+
     include CheckSupport
     extend CheckSupport::ClassMethods
 
@@ -26,7 +28,8 @@ module Shiba
       end
 
       @options = options
-      @explain_json = Shiba.connection.explain(@sql)
+
+      @explain_json, @select_fields = Shiba.connection.explain(@sql)
 
       if Shiba.connection.mysql?
         @rows = Shiba::Explain::MysqlExplain.new.transform_json(@explain_json['query_block'])
@@ -149,18 +152,30 @@ module Shiba
       end
     end
 
+    def select_row_size
+      size = 0
+      @select_fields.each do |table, fields|
+        fields.each do |f|
+          size += @stats.get_column_size(table, f) || 0
+        end
+      end
+      size
+    end
+
     def check_return_size
       if @query.limit
-        return_size = [@query.limit, @result.result_size].min
+        result_size = [@query.limit, @result.result_size].min
       elsif @query.aggregation?
-        return_size = 1
+        result_size = 1
       else
-        return_size = @result.result_size
+        result_size = @result.result_size
       end
 
-      cost = COST_PER_ROW_RETURNED * return_size
+      result_bytes = select_row_size * result_size
+      cost = (result_bytes / 1024.0) * COST_PER_KB_RETURNED
+
       @result.cost += cost
-      @result.messages << { tag: "retsize", result_size: return_size, cost: cost }
+      @result.messages << { tag: "retsize", result_size: result_size, result_bytes: result_bytes, cost: cost }
     end
 
     def run_checks!
@@ -191,7 +206,7 @@ module Shiba
     def other_paths
       if Shiba.connection.mysql?
         @rows.map do |r|
-          next [] unless r['possible_keys'] && r['key'].nil?
+          next [] unless r['possible_keys']
           possible = r['possible_keys'] - [r['key']]
           possible.map do |p|
             Explain.new(@query, @stats, force_key: p) rescue nil

data/lib/shiba/fuzzer.rb

@@ -35,6 +35,11 @@ module Shiba
 
         stats.add_index_column(h['table_name'], h['index_name'], h['column_name'], h['cardinality'], h['non_unique'] == 0)
       end
+
+      connection.each_column_size do |table, column, size|
+        stats.set_column_size(table, column, size)
+      end
+
       stats
     end