shiba 0.2.3 → 0.3.0

data/lib/shiba/activerecord_integration.rb

@@ -1,12 +1,13 @@
 require 'shiba/query_watcher'
 require 'active_support/notifications'
 require 'active_support/lazy_load_hooks'
+require 'shiba/configure'
 
 module Shiba
   # Integrates ActiveRecord with the Query Watcher by setting up the query log path, and the
   # connection options for the explain command, which it runs when the process exits.
   #
-  # SHIBA_OUT=<log path> and SHIBA_DEBUG=true environment variables may be set.
+  # SHIBA_OUT and SHIBA_DEBUG=true environment variables may be set.
   class ActiveRecordIntegration
 
     attr_reader :path, :watcher
@@ -24,12 +25,11 @@ module Shiba
     protected
 
     def self.start_watcher
+      path = log_path
       if ENV['SHIBA_DEBUG']
-        $stderr.puts("starting shiba watcher")
+        $stderr.puts("starting shiba watcher, outputting to #{path}")
       end
 
-      path = ENV['SHIBA_OUT'] || make_tmp_path
-
       file = File.open(path, 'a')
       watcher = QueryWatcher.new(file)
 
@@ -40,14 +40,20 @@ module Shiba
       $stderr.puts(e.message, e.backtrace.join("\n"))
     end
 
-    def self.make_tmp_path
-      "/tmp/shiba-query.log-#{Time.now.to_i}"
+    def self.log_path
+      name = ENV["SHIBA_OUT"] || "query.log-#{Time.now.to_i}"
+      File.join(Shiba.path, name)
     end
 
     def self.run_explain(file, path)
       file.close
       puts ""
+
       cmd = "shiba explain #{database_args} --file #{path}"
+      if Shiba::Configure.ci?
+        cmd << " --json #{File.join(Shiba.path, 'ci.json')}"
+      end
+
       if ENV['SHIBA_DEBUG']
         $stderr.puts("running:")
         $stderr.puts(cmd)
@@ -56,16 +62,27 @@ module Shiba
     end
 
     def self.database_args
-      c = ActiveRecord::Base.connection.raw_connection.query_options
+      cx = ActiveRecord::Base.connection.raw_connection
+      if cx.respond_to?(:query_options)
+        # mysql
+        c = cx.query_options.merge(server: 'mysql')
+      else
+        # postgres
+        c = { host: cx.host, database: cx.db, username: cx.user, password: cx.pass, port: cx.port, server: 'postgres' }
+      end
+
       options = {
-      'host':     c[:host],
-      'database': c[:database],
-      'user':     c[:username],
-      'password': c[:password]
+        'host':     c[:host],
+        'database': c[:database],
+        'user':     c[:username],
+        'password': c[:password],
+        'port':     c[:port],
+        'server':   c[:server]
       }
 
-      options.reject { |k,v| v.nil? }.map { |k,v| "--#{k} #{v}" }.join(" ")
+      # port can be a Fixnum
+      options.reject { |k,v| v.nil? || v.respond_to?(:empty?) && v.empty? }.map { |k,v| "--#{k} #{v}" }.join(" ")
     end
 
   end
-end
+end

data/lib/shiba/checker.rb

@@ -5,7 +5,11 @@ require 'shiba/diff'
 require 'shiba/backtrace'
 
 module Shiba
+  # Given an explain log and a diff, returns any explain logs
+  # that appear to be caused by the diff.
   class Checker
+    MAGIC_COST = 100
+
     Result = Struct.new(:status, :message, :problems)
 
     attr_reader :options
@@ -14,6 +18,8 @@ module Shiba
       @options = options
     end
 
+    # Returns a Result object with a status, message, and any problem queries detected.
+    # Query problem format is [ [ "path:lineno", explain ]... ]
     def run(log)
       msg = nil
 
@@ -21,9 +27,9 @@ module Shiba
         puts cmd
       end
 
-      if changes.empty?
+      if changed_files.empty?
         if options['verbose']
-          msg = "No changes found in git"
+          msg = "No changes found. Are you sure you specified the correct branch?"
         end
         return Result.new(:pass, msg)
       end
@@ -31,15 +37,27 @@ module Shiba
       explains = select_lines_with_changed_files(log)
       problems = explains.select { |explain| explain["cost"] && explain["cost"] > MAGIC_COST }
 
-      problems.select! do |problem|
-        backtrace_has_updated_line?(problem["backtrace"], updated_lines)
+
+      if options["verbose"]
+        puts problems
+        puts "Updated lines: #{updated_lines}"
       end
 
       if problems.empty?
-        if options['verbose']
-          msg = "No problems found"
-        end
+        msg = "No problems found caused by the diff"
+        return Result.new(:pass, msg)
+      end
+
+      problems.map! do |problem|
+        line = updated_line_from_backtrace(problem["backtrace"], updated_lines)
+        next if line.nil?
+
+        [ line, problem ]
+      end
+      problems.compact!
 
+      if problems.empty?
+        msg = "No problems found caused by the diff"
         return Result.new(:pass, msg)
       end
 
@@ -48,45 +66,91 @@ module Shiba
 
     protected
 
-    def backtrace_has_updated_line?(backtrace, updates)
-      backtrace.any? do |bl|
-        updates.any? do |path, lines|
+    def updated_line_from_backtrace(backtrace, updates)
+      backtrace.each do |bl|
+        updates.each do |path, lines|
           next if !bl.start_with?(path)
           bl =~ /:(\d+):/
-          lines.include?($1.to_i)
+          next if !lines.include?($1.to_i)
+
+          return "#{path}:#{$1}"
         end
       end
+
+      return nil
     end
 
     def select_lines_with_changed_files(log)
-      patterns = changes.split("\n").map { |path| "-e #{path}" }.join(" ")
-      json_lines = `grep #{log} #{patterns}`
+      patterns = changed_files.map { |path| "-e #{path}" }.join(" ")
+      cmd = "grep #{log} #{patterns}"
+      $stderr.puts cmd if options["verbose"]
+
+      json_lines = `#{cmd}`
       json_lines.each_line.map { |line| JSON.parse(line) }
     end
 
-    def changes
-      @changes ||= begin
-        result = `git diff#{cmd} --name-only --diff-filter=d`
-        if $?.exitstatus != 0
-          error("Failed to read changes", $?.exitstatus)
-        end
-
-        result
+    def changed_files
+      @changed_files ||= begin
+        options['diff'] ? file_diff_names : git_diff_names
       end
     end
 
     def updated_lines
       return @updated_lines if @updated_lines
 
-      Open3.popen3("git diff#{cmd} --unified=0 --diff-filter=d") {|_,o,_,_|
-        @updated_lines = Shiba::Diff.new(o).updated_lines
-      }
+
+      out = options['diff'] ? file_diff_lines : git_diff_lines
+      @updated_lines = Shiba::Diff.new(out).updated_lines
+
 
       @updated_lines.map! do |path, lines|
         [ Shiba::Backtrace.clean!(path), lines ]
       end
     end
 
+    def file_diff_lines
+      File.open(options['diff'])
+    end
+
+    def git_diff_lines
+      run = "git diff#{cmd} --unified=0 --diff-filter=d"
+      if options[:verbose]
+        $stderr.puts run
+      end
+
+      _, out,_,_ = Open3.popen3(run)
+      out
+    end
+
+    # index ade9b24..661d522 100644
+    # --- a/test/app/app.rb
+    # +++ b/test/app/app.rb
+    # @@ -24,4 +24,4 @@ ActiveRecord::Base...
+    # org = Organization.create!(name: 'test')
+    #
+    # file_diff_lines
+    # => test/app/app.rb
+    def file_diff_names
+      file_name_pattern = /^\+\+\+ b\/(.*?)$/
+      f = File.open(options['diff'])
+      f.grep(file_name_pattern) { $1 }
+    end
+
+    def git_diff_names
+      run = "git diff#{cmd} --name-only --diff-filter=d"
+
+      if options[:verbose]
+        $stderr.puts run
+      end
+      result = `#{run}`
+      if $?.exitstatus != 0
+        $stderr.puts result
+        raise Shiba::Error.new "Failed to read changes"
+      end
+
+      result.split("\n")
+    end
+
     def cmd
       cmd = case
       when options["staged"]
@@ -94,7 +158,7 @@ module Shiba
       when options["unstaged"]
         ""
       else
-        commit = " HEAD"
+        commit = " origin/HEAD"
         commit << "...#{options["branch"]}" if options["branch"]
         commit
       end

data/lib/shiba/configure.rb

@@ -1,5 +1,7 @@
 require 'pathname'
 require 'pp'
+require 'optionparser'
+
 module Shiba
   module Configure
 
@@ -27,6 +29,10 @@ module Shiba
       raise e, "Cannot load `#{path}`:\n#{e.message}", e.backtrace
     end
 
+    def self.ci?
+      ENV['CI'] || ENV['CONTINUOUS_INTEGRATION']
+    end
+
     # loosely based on https://dev.mysql.com/doc/refman/8.0/en/option-files.html
     def self.mysql_config_path
       paths = [ File.join(Dir.home, '.mylogin.cnf'), File.join(Dir.home, '.my.cnf')  ]
@@ -53,10 +59,14 @@ module Shiba
       end
     end
 
-    def self.make_options_parser(options)
-      parser = OptionParser.new do |opts|
+    def self.make_options_parser(options, only_basics = false)
+      OptionParser.new do |opts|
         # note that the key to the hash needs to stay the same as the
         # option name since we re-pass them
+        opts.on("-s","--server SERVER_TYPE", "mysql|postgres") do |s|
+          options["server"] = s
+        end
+
         opts.on("-h","--host HOST", "sql host") do |h|
           options["host"] = h
         end
@@ -73,6 +83,10 @@ module Shiba
           options["password"] = p
         end
 
+        opts.on("-P","--port PORT", "server port") do |p|
+          options["port"] = p
+        end
+
         opts.on("-c","--config FILE", "location of shiba.yml") do |f|
           options["config"] = f
         end
@@ -81,6 +95,12 @@ module Shiba
           options["index"] = i.to_i
         end
 
+        opts.on("--default-extras-file", "The option file to read mysql configuration from") do |f|
+          options["default_file"] = f
+        end
+
+        next if only_basics
+
         opts.on("-l", "--limit NUM", "stop after processing NUM queries") do |l|
           options["limit"] = l.to_i
         end
@@ -114,9 +134,6 @@ module Shiba
           options["default_group"] = f
         end
 
-        opts.on("--default-extras-file", "The option file to read mysql configuration from") do |f|
-          options["default_file"] = f
-        end
       end
     end
   end

data/lib/shiba/connection.rb

@@ -0,0 +1,25 @@
+module Shiba
+  class Connection
+    def self.build(hash)
+      server_type = hash['server']
+      if !server_type
+        port = hash['port'].to_i
+        if port == 3306
+          server_type = 'mysql'
+        elsif port == 5432
+          server_type = 'postgres'
+        else
+          raise "couldn't determine server type!  please pass --server"
+        end
+      end
+
+      if server_type == 'mysql'
+        require 'shiba/connection/mysql'
+        Shiba::Connection::Mysql.new(hash)
+      else
+        require 'shiba/connection/postgres'
+        Shiba::Connection::Postgres.new(hash)
+      end
+    end
+  end
+end

data/lib/shiba/connection/mysql.rb

@@ -0,0 +1,45 @@
+require 'mysql2'
+require 'json'
+
+module Shiba
+  class Connection
+    class Mysql
+      def initialize(hash)
+        @connection = Mysql2::Client.new(hash)
+      end
+
+      def query(sql)
+        @connection.query(sql)
+      end
+
+      def fetch_indexes
+        sql =<<-EOL
+          select * from information_schema.statistics where
+          table_schema = DATABASE()
+          order by table_name, if(index_name = 'PRIMARY', '', index_name), seq_in_index
+        EOL
+        @connection.query(sql)
+      end
+
+      def count_indexes_by_table
+        sql =<<-EOL
+          select TABLE_NAME as table_name, count(*) as index_count
+          from information_schema.statistics where table_schema = DATABASE()
+          and seq_in_index = 1 and index_name not like 'fk_rails%'
+          group by table_name order by index_count
+        EOL
+
+        @connection.query(sql).to_a
+      end
+
+      def explain(sql)
+        rows = query("EXPLAIN FORMAT=JSON #{sql}").to_a
+        JSON.parse(rows.first['EXPLAIN'])
+      end
+
+      def mysql?
+        true
+      end
+    end
+  end
+end

data/lib/shiba/connection/postgres.rb

@@ -0,0 +1,91 @@
+require 'pg'
+
+module Shiba
+  class Connection
+    class Postgres
+      def initialize(h)
+        @connection = PG.connect( dbname: h['database'], host: h['host'], user: h['username'], password: h['password'], port: h['port'] )
+        @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
+        query("SET enable_seqscan = OFF")
+      end
+
+      def query(sql)
+        @connection.query(sql)
+      end
+
+      def fetch_indexes
+        result = query(<<-EOL
+          select
+              t.relname as table_name,
+              i.relname as index_name,
+              a.attname as column_name,
+              i.reltuples as numrows,
+              ix.indisunique as is_unique,
+              ix.indisprimary as is_primary,
+              s.n_distinct as numdistinct
+          from pg_namespace p
+          join pg_class t on t.relnamespace = p.oid
+          join pg_index ix on ix.indrelid = t.oid
+          join pg_class i on i.oid = ix.indexrelid
+          join pg_attribute a on a.attrelid = t.oid
+          left join pg_stats s on s.tablename = t.relname
+              AND s.attname = a.attname
+          where
+              p.nspname = 'public'
+              and a.attnum = ANY(ix.indkey)
+              and t.relkind = 'r'
+          order by
+              t.relname,
+              ix.indisprimary desc,
+              i.relname,
+              array_position(ix.indkey, a.attnum)
+          EOL
+        )
+        rows = result.to_a.map do |row|
+          # TBD: do better than this, have them return something objecty
+          if row['is_primary'] == "t"
+            row['index_name'] = "PRIMARY"
+            row['non_unique'] = 0
+          elsif row['is_unique']
+            row['non_unique'] = 0
+          end
+
+          if row['numdistinct'].nil?
+            # meaning the table's empty.
+            row['cardinality'] = 0
+          elsif row['numdistinct'] == 0
+            # numdistinct is 0 if there's rows in the table but all values are null
+            row['cardinality'] = 1
+          elsif row['numdistinct'] < 0
+            # postgres talks about either cardinality or selectivity (depending.  what's their heuristic?)
+            # in the same way we do in the yaml file!
+            # if less than zero, it's negative selectivity.
+            row['cardinality'] = -(row['numrows'] * row['numdistinct'])
+          else
+            row['cardinality'] = row['numdistinct']
+          end
+          row
+        end
+
+        #TODO: estimate multi-index column cardinality
+        rows
+      end
+
+      def count_indexes_by_table
+        sql = <<-EOL
+          select tablename as table_name, count(*) as index_count from pg_indexes where schemaname='public' group by 1 order by 2
+        EOL
+        @connection.query(sql).to_a
+      end
+
+      def explain(sql)
+        rows = query("EXPLAIN (FORMAT JSON) #{sql}").to_a
+        rows.first["QUERY PLAN"]
+      end
+
+      def mysql?
+        false
+      end
+    end
+  end
+end