shiba 0.5.0 → 0.6.0

data/lib/shiba/index_stats.rb

@@ -13,7 +13,7 @@ module Shiba
       @tables.any?
     end
 
-    Table = Struct.new(:name, :count, :indexes, :average_row_size) do
+    Table = Struct.new(:name, :count, :indexes) do
       def encode_with(coder)
         coder.map = self.to_h.stringify_keys
         coder.map.delete('name')
@@ -23,6 +23,7 @@ module Shiba
           self.count = indexes.map { |i, parts| parts.columns.map { |v| v.raw_cardinality } }.flatten.max
         end
 
+        coder.map['column_sizes'] = column_sizes
         coder.tag = nil
       end
 
@@ -39,6 +40,10 @@ module Shiba
           self.count = cardinality
         end
       end
+
+      def column_sizes
+        @column_sizes ||= {}
+      end
     end
 
     Index = Struct.new(:table, :name, :columns, :unique) do
@@ -152,6 +157,20 @@ module Shiba
       table.add_index_column(index_name, column_name, nil, cardinality, is_unique)
     end
 
+    def get_column_size(table_name, column)
+      table = @tables[table_name]
+      return nil unless table
+
+      table.column_sizes[column]
+    end
+
+    def set_column_size(table_name, column, size)
+      table = @tables[table_name]
+      raise "couldn't find table: #{table_name}" unless table
+
+      table.column_sizes[column] = size
+    end
+
     def estimate_key(table_name, key, parts)
       index = fetch_index(table_name, key)
 

data/lib/shiba/output/tags.yaml

@@ -71,4 +71,4 @@ index_walk:
   level: success
 retsize:
   title: Results
-  summary: The database returns {{ result_size }} row(s) to the client.
+  summary: The database returns {{ formatted_result }} to the client.

data/lib/shiba/parsers/mysql_select_fields.rb

@@ -0,0 +1,64 @@
+require 'shiba/parsers/shiba_string_scanner'
+
+module Shiba
+  module Parsers
+    class MysqlSelectFields
+      def initialize(sql)
+        @sql = sql
+        @sc = ShibaStringScanner.new(@sql)
+      end
+      attr_reader :sc
+
+      BACKTICK = "`"
+
+      def tick_match
+        sc.match_quoted_double_escape(BACKTICK)
+      end
+
+      def parse_fields
+        tables = {}
+
+        sc.scan(%r{/\*.*?\*/ select })
+
+        while !sc.scan(/ from/i)
+          sc.scan(/distinct /)
+
+          if sc.scan(/\w+\(/)
+            parens = 1
+            while parens > 0
+              case sc.getch
+              when '('
+                parens += 1
+              when ')'
+                parens -= 1
+              end
+            end
+            sc.scan(/ AS /)
+            tick_match
+            # parse function
+          elsif sc.scan(/`(.*?)`\.`(.*?)`\.`(.*?)` AS `(.*?)`/)
+            db = sc[1]
+            table = sc[2]
+            col = sc[3]
+
+            tables[table] ||= []
+            tables[table] << col
+
+          elsif sc.scan(/`(.*?)`\.`(.*?)` AS `(.*?)`/)
+            table = sc[1]
+            col = sc[2]
+
+            tables[table] ||= []
+            tables[table] << col
+          elsif sc.scan(/(\d+|NULL|'.*?') AS `(.*?)`/m)
+          else
+            raise "unknown stuff: in #{@sql}: #{@sc.rest}"
+          end
+
+          sc.scan(/,/)
+        end
+        tables
+      end
+    end
+  end
+end

data/lib/shiba/parsers/shiba_string_scanner.rb

@@ -0,0 +1,27 @@
+require 'strscan'
+
+module Shiba
+  module Parsers
+    class ShibaStringScanner < StringScanner
+      def match_quoted_double_escape(quote)
+        getch
+
+        str = ""
+        while ch = getch
+          if ch == quote
+            if peek(1) == quote
+              str += ch
+              str += getch
+            else
+              return str
+            end
+          else
+            str += ch
+          end
+        end
+        str
+      end
+    end
+  end
+end
+

data/lib/shiba/review/cli.rb

@@ -0,0 +1,196 @@
+module Shiba
+  module Review
+    # Builds options for interacting with the reviewer via the command line.
+    # Automatically infers options from environment variables on CI.
+    #
+    # Example:
+    # cli = CLI.new
+    # cli.valid?
+    # => true
+    # cli.options
+    # => { "file" => 'path/to/explain_log.json' }
+    #
+    # or
+    #
+    # cli.valid?
+    # => false
+    # cli.failure
+    # => "An error message with command line help."
+    class CLI
+
+      attr_reader :errors
+
+      def initialize
+        @user_options = {}
+        @errors = []
+        parser.parse!
+        @options = default_options.merge(@user_options)
+      end
+
+      def options
+        @options
+      end
+
+      def valid?
+        return false if @errors.any?
+
+        validate_log_path
+        #validate_git_repo if branch || options["submit"]
+        description = "Provide an explain log, or run 'shiba explain' to generate one."
+
+        require_option("file", description: description)
+
+        if options["submit"]
+          require_option("branch") if options["diff"].nil?
+          require_option("token")
+          require_option("pull_request")
+        end
+
+        @errors.empty?
+      end
+
+      def failure
+        return nil if @errors.empty?
+
+        message, help = @errors.first
+        message += "\n"
+        if help
+          message += "\n#{parser}"
+        end
+
+        message
+      end
+
+      def report_options(*keys)
+        keys.each do |key|
+          report("#{key}: #{options[key]}")
+        end
+      end
+
+      protected
+
+      def parser
+        @parser ||= OptionParser.new do |opts|
+          opts.banner = "Review changes for query problems. Optionally submit the comments to a Github pull request."
+
+          opts.separator "Required:"
+
+          opts.on("-f","--file FILE", "The explain output log to compare with. Automatically configured when $CI environment variable is set") do |f|
+            @user_options["file"] = f
+          end
+
+          opts.separator ""
+          opts.separator "Git diff options:"
+
+          opts.on("-b", "--branch GIT_BRANCH", "Compare to changes between origin/HEAD and BRANCH. Attempts to read from CI environment when not set.") do |b|
+            @user_options["branch"] = b
+          end
+
+          opts.on("--staged", "Only check files that are staged for commit") do
+            @user_options["staged"] = true
+          end
+
+          opts.on("--unstaged", "Only check files that are not staged for commit") do
+            @user_options["unstaged"] = true
+          end
+
+          opts.separator ""
+          opts.separator "Github options:"
+
+          opts.on("--submit", "Submit comments to Github") do
+            @user_options["submit"] = true
+          end
+
+          opts.on("-p", "--pull-request PR_ID", "The ID of the pull request to comment on. Attempts to read from CI environment when not set.") do |p|
+            @user_options["pull_request"] = p
+          end
+
+          opts.on("-t", "--token TOKEN", "The Github API token to use for commenting. Defaults to $GITHUB_TOKEN.") do |t|
+            @user_options["token"] = t
+          end
+
+          opts.separator ""
+          opts.separator "Common options:"
+
+          opts.on("--verbose", "Verbose/debug mode") do
+            @user_options["verbose"] = true
+          end
+
+          opts.on_tail("-h", "--help", "Show this message") do
+            puts opts
+            exit
+          end
+
+          opts.on_tail("--version", "Show version") do
+            require 'shiba/version'
+            puts Shiba::VERSION
+            exit
+          end
+        end
+      end
+
+      def default_options
+        defaults = {}
+
+        if ENV['DIFF']
+          defaults['diff'] = ENV['DIFF']
+        end
+
+        if Shiba::Configure.ci?
+          report("Finding default options from CI environment.")
+
+          defaults["file"]         = ci_explain_log_path
+          defaults["pull_request"] = ci_pull_request
+          defaults["branch"]       = ci_branch if !defaults['diff']
+        end
+
+        defaults["token"] = ENV['GITHUB_TOKEN'] if ENV['GITHUB_TOKEN']
+
+        defaults
+      end
+
+      def validate_log_path
+        return if !options["file"]
+        if !File.exist?(options["file"])
+          error("File not found: '#{options["file"]}'")
+        end
+      end
+
+      def ci_explain_log_path
+        File.join(Shiba.path, 'ci.json')
+      end
+
+      def ci_branch
+        ENV['TRAVIS_PULL_REQUEST_SHA'] || ENV['CIRCLE_SHA1']
+      end
+
+      def ci_pull_request
+        ENV['TRAVIS_PULL_REQUEST'] || circle_pr_number
+      end
+
+      def circle_pr_number
+        return if ENV["CIRCLE_PULL_REQUEST"].nil?
+        number = URI.parse(ENV["CIRCLE_PULL_REQUEST"]).path.split("/").last
+        return if number !~ /\A[0-9]+\Z/
+
+        number
+      end
+
+      def require_option(name, description: nil)
+        return true if options.key?(name)
+        msg = "Required: '#{name}'"
+        msg << ". #{description}" if description
+        error(msg, help: true)
+      end
+
+      def report(message)
+        $stderr.puts message if @user_options["verbose"]
+      end
+
+      def error(message, help: false)
+        @errors << [ message, help ]
+      end
+
+    end
+  end
+end

data/lib/shiba/review/comment_renderer.rb

@@ -43,15 +43,32 @@ module Shiba
           "fuzz_table_sizes" => fuzzed_sizes(message),
           "table"           => message["table"],
           "table_size"      => message["table_size"],
-          "result_size"     => message["result_size"],
           "index"           => message["index"],
           "join_to"         => message["join_to"],
           "key_parts"       => (message["index_used"] || []).join(','),
           "size"            => message["size"],
-          "formatted_cost"  => formatted_cost(message)
+          "formatted_cost"  => formatted_cost(message),
+          "formatted_result" => formatted_result(message)
         }
       end
 
+      def formatted_result(explain)
+        return nil unless explain['result_bytes'] && explain['result_size']
+
+        bytes = explain['result_bytes']
+        return "%d rows" % explain['result_size'] if bytes == 0
+
+        if bytes < 1000
+          result = "%d bytes" % bytes
+        elsif bytes < 1000000
+          result = "%dkb" % (bytes / 1000)
+        else
+          result = "%.1fmb" % (bytes / 1000000.0 )
+        end
+
+        "%s (%d rows)" % [result, explain['result_size']]
+      end
+
       def formatted_cost(explain)
         return nil unless explain["rows_read"] && explain["table_size"]
         percentage = (explain["rows_read"] / explain["table_size"]) * 100.0;

data/lib/shiba/review/diff.rb

@@ -0,0 +1,227 @@
+module Shiba
+  module Review
+    module Diff
+
+      class Parser
+        # +++ b/config/environments/test.rb
+        FILE_PATTERN = /\A\+\+\+ b\/(.*?)\Z/
+
+        # @@ -177,0 +178 @@ ...
+        # @@ -177,0 +178,5 @@ ...
+        # @@ -21 +24 @@ ...
+        LINE_PATTERN = /\A@@ \-\d+,?\d+? \+(\d+),?(\d+)? @@/
+
+        # via https://developer.github.com/v3/pulls/comments/#create-a-comment
+        # The position value equals the number of lines down from the first "@@" hunk header
+        # in the file you want to add a comment.
+
+        attr_reader :status
+
+        def initialize(file)
+          # Fixme. seems like enumerables should work in general.
+          if !file.respond_to?(:pos)
+            raise StandardError.new("Diff file does not appear to be a seekable IO object.")
+          end
+          @diff = file
+          @status = :new
+        end
+
+        # Returns the file and line numbers that contain inserts. Deletions are ignored.
+        # For simplicity, the default output of git diff is not supported.
+        # The expected format is from 'git diff unified=0'
+        #
+        # Example:
+        # diff = `git diff --unified=0`
+        # Diff.new(StringIO.new(diff))
+        # => [ [ "hello.rb", 1..3 ]
+        # =>   [ "hello.rb", 7..7 ]
+        # =>   [ "test.rb", 23..23 ]
+        # => ]
+        def updated_lines
+          io = @diff.each_line
+          path = nil
+
+          found = []
+
+          while true
+            line = io.next
+            if line =~ FILE_PATTERN
+              path = $1
+            end
+
+            if hunk_header?(line)
+              line_numbers = line_numbers_for_destination(line)
+              found << [ path, line_numbers ]
+            end
+          end
+        rescue StopIteration
+          return found
+        end
+
+        # Returns the position in the diff, after the relevant file header,
+        # that contains the specified file/lineno modification.
+        # Only supports finding the position in the destination / newest version of the file.
+        #
+        # Example:
+        # diff = Diff.new(`git diff`)
+        # diff.find_position("test.rb", 3)
+        # => 5
+        def find_position(path, line_number)
+          io = @diff.each_line # maybe redundant?
+
+          file_header = "+++ b/#{path}\n" # fixme
+          if !io.find_index(file_header)
+            @status = :file_not_found
+            return
+          end
+
+          line = io.peek
+          if !hunk_header?(line)
+            raise StandardError.new("Expected hunk header to be after file header, but got '#{line}'")
+          end
+
+          pos = 0
+
+          while true
+            line = io.next
+            pos += 1
+
+            if file_header?(line)
+              @status = :line_not_found
+              return
+            end
+
+            if !hunk_header?(line)
+              next
+            end
+
+            line_numbers = line_numbers_for_destination(line)
+
+            if destination_position = line_numbers.find_index(line_number)
+              @status = :found_position
+              return pos + find_hunk_index(io, destination_position)
+            end
+          end
+        rescue StopIteration
+          @status = :line_not_found
+        end
+
+        protected
+
+        def find_hunk_index(hunk, pos)
+          line, idx = hunk.with_index.select { |l,idx| !l.start_with?('-') }.take(pos+1).last
+          idx
+        end
+
+        def file_header?(line)
+          line =~ FILE_PATTERN
+        end
+
+        def hunk_header?(line)
+          LINE_PATTERN =~ line
+        end
+
+        def line_numbers_for_destination(diff_line)
+          diff_line =~ LINE_PATTERN
+          line = $1.to_i
+          line_count = ($2 && $2.to_i) || 0
+          line..line+line_count
+        end
+      end
+
+      class GitDiff
+
+        # Valid options are: "staged", "unstaged", "branch", and "verbose"
+        def initialize(options)
+          @options = options
+        end
+
+        def paths
+          if options['verbose']
+            puts cmd
+          end
+
+          run = "git diff#{cmd} --name-only --diff-filter=d"
+
+          if options[:verbose]
+            $stderr.puts run
+          end
+          result = `#{run}`
+          if $?.exitstatus != 0
+            $stderr.puts result
+            raise Shiba::Error.new "Failed to read changes"
+          end
+
+          result.split("\n")
+        end
+
+        # Differ expects context: 0, ignore_deletions: true
+        def file(context: nil, ignore_deletions: false)
+          differ = "git diff#{cmd}"
+          differ << " --unified=#{context}" if context
+          differ << " --diff-filter=d"      if ignore_deletions
+          run(differ)
+        end
+
+        protected
+
+        attr_reader :options
+
+        def run(command)
+          if options[:verbose]
+            $stderr.puts command
+          end
+
+          _, out,_,_ = Open3.popen3(command)
+          out
+        end
+
+        def cmd
+          cmd = case
+          when options["staged"]
+            " --staged"
+          when options["unstaged"]
+            ""
+          else
+            commit = " origin/HEAD"
+            commit << "...#{options["branch"]}" if options["branch"]
+            commit
+          end
+        end
+      end
+
+      class FileDiff
+        # +++ b/test/app/app.rb
+        FILE_NAME_PATTERN = /^\+\+\+ b\/(.*?)$/
+
+        def initialize(path)
+          @path = path
+        end
+
+        # Extracts path names from the diff file
+        #
+        # Example:
+        # index ade9b24..661d522 100644
+        # --- a/test/app/app.rb
+        # +++ b/test/app/app.rb
+        # @@ -24,4 +24,4 @@ ActiveRecord::Base...
+        # org = Organization.create!(name: 'test')
+        #
+        # diff.paths
+        # => [ test/app/app.rb ]
+        def paths
+          f = File.open(@path)
+          f.grep(FILE_NAME_PATTERN) { $1 }
+        end
+
+        def file(context: nil, ignore_deletions: nil)
+          warn "Context not supported for file diffs" if context
+          warn "Ignore deletions not supported for file diffs" if ignore_deletions
+          File.open(@path)
+        end
+
+      end
+
+    end
+  end
+end