shiba 0.5.0 → 0.6.0

data/lib/shiba/review/explain_diff.rb

@@ -0,0 +1,117 @@
+require 'json'
+require 'open3'
+
+require 'shiba/review/diff'
+require 'shiba/backtrace'
+
+module Shiba
+  module Review
+    # Given an explain log and a diff, returns any explain logs
+    # that appear to be caused by the diff.
+    class ExplainDiff
+      Result = Struct.new(:status, :message)
+
+      attr_reader :options
+
+      def initialize(log, options)
+        @log     = log
+        @options = options
+      end
+
+      def diff_requested_by_user?
+        [ "staged", "unstaged", "branch", "diff" ].any? { |key| @options.key?(key) }
+      end
+
+      # Returns detected problem queries with their line numbers.
+      # Query problem format is [ [ "path:lineno", explain ]... ]
+      def problems
+        return @problems if @problems
+
+        @problems = explains_with_backtrace_in_diff.select do |explain|
+          explain["severity"] && explain["severity"] != 'none'
+        end
+
+        if options["verbose"]
+          $stderr.puts @problems
+          $stderr.puts "Updated lines: #{updated_lines}"
+        end
+
+        @problems.map! do |problem|
+          line = diff_line_from_backtrace(problem["backtrace"])
+          next if line.nil?
+
+          [ line, problem ]
+        end
+        @problems.compact!
+
+        @problems
+      end
+
+      def result
+        msg = nil
+
+        if changed_files.empty?
+          if options['verbose']
+            msg = "No changes found. Are you sure you specified the correct branch?"
+          end
+          return Result.new(:pass, msg)
+        end
+
+        if problems.empty?
+          msg = "No problems found caused by the diff"
+          return Result.new(:pass, msg)
+        end
+
+        return Result.new(:fail, "Potential problems")
+      end
+
+      protected
+
+      # file.rb:32:in `hello'",
+      LINE_NUMBER_PATTERN = /:(\d+):/
+
+      def diff_line_from_backtrace(backtrace)
+        backtrace.each do |bl|
+          updated_lines.each do |path, lines|
+            next if !bl.start_with?(path)
+            bl =~ LINE_NUMBER_PATTERN
+            next if !lines.include?($1.to_i)
+
+            return "#{path}:#{$1}"
+          end
+        end
+
+        return nil
+      end
+
+      # All explains from the log file with a backtrace that contains a changed file.
+      def explains_with_backtrace_in_diff
+        patterns = changed_files.map { |path| "-e #{path}" }.join(" ")
+        cmd = "grep #{@log} #{patterns}"
+        $stderr.puts cmd if options["verbose"]
+
+        json_lines = `#{cmd}`
+        json_lines.each_line.map { |line| JSON.parse(line) }
+      end
+
+      def changed_files
+        @changed_files ||= diff.paths
+      end
+
+      def updated_lines
+        return @updated_lines if @updated_lines
+
+        diff_file = diff.file(context: 0, ignore_deletions: true)
+        @updated_lines = Diff::Parser.new(diff_file).updated_lines
+        @updated_lines.map! do |path, lines|
+          [ Shiba::Backtrace.clean!(path), lines ]
+        end
+      end
+
+      def diff
+        @diff ||= options["diff"] ? Diff::FileDiff.new(options["diff"]) : Diff::GitDiff.new(options)
+      end
+
+    end
+  end
+end

data/lib/shiba/reviewer.rb

@@ -1,6 +1,6 @@
 require 'open3'
 require 'shiba'
-require 'shiba/diff'
+require 'shiba/review/diff'
 require 'shiba/review/api'
 require 'shiba/review/comment_renderer'
 
@@ -10,7 +10,6 @@ module Shiba
   # 2. May make sense to edit the comment on a commit line when the code
   # is semi-corrected but still a problem
   class Reviewer
-    TEMPLATE_FILE = File.join(Shiba.root, 'lib/shiba/output/tags.yaml')
     MESSAGE_FILTER_THRESHOLD = 0.005
 
     attr_reader :repo_url, :problems, :options
@@ -20,7 +19,9 @@ module Shiba
       @problems = problems
       @options = options
       @commit_id = options.fetch("branch") do
-        raise Shiba::Error.new("Must specify a branch") if !options['diff']
+        if options["submit"]
+          raise Shiba::Error.new("Must specify a branch")
+        end
       end
     end
 
@@ -33,7 +34,11 @@ module Shiba
           raise Shiba::Error.new("Bad path received: #{line_number}")
         end
 
-        position = diff.find_position(file, line_number.to_i)
+        position = if path == "none:-1"
+          nil
+        else
+          diff_parser.find_position(file, line_number.to_i)
+        end
 
         explain = keep_only_dangerous_messages(explain)
 
@@ -97,22 +102,18 @@ module Shiba
       explain_b
     end
 
-    def diff
-      return @diff if @diff
-      output = options['diff'] ? file_diff : git_diff
-      @diff = Shiba::Diff.new(output)
-    end
-
-    def git_diff
-      cmd ="git diff origin/HEAD..#{@commit_id}"
-      report("Finding PR position using: #{cmd}")
-
-      output = StringIO.new(`#{cmd}`)
+    def diff_parser
+      @diff_parser ||= Review::Diff::Parser.new(diff.file)
     end
 
-    def file_diff
-      report("Finding PR position using file: #{options['diff']}")
-      File.open(options['diff'], 'r')
+    def diff
+      @diff ||= if options['diff']
+        report("Finding PR position using file: #{options['diff']}")
+        Review::Diff::FileDiff.new(options['diff'])
+      else
+        report("Finding PR position using git")
+        Review::Diff::GitDiff.new(options)
+      end
     end
 
     def api
@@ -130,7 +131,7 @@ module Shiba
     end
 
     def tags
-      @tags ||=  YAML.load_file(TEMPLATE_FILE)
+      @tags ||= YAML.load_file(Shiba::TEMPLATE_FILE)
     end
 
   end

data/lib/shiba/table_stats.rb

@@ -18,6 +18,10 @@ module Shiba
       ask_each(:table_count, table)
     end
 
+    def get_column_size(table_name, column)
+      ask_each(:get_column_size, table_name, column)
+    end
+
     def fuzzed?(table)
       !@dump_stats.tables[table] && !@manual_stats.tables[table] && @db_stats.tables[table]
     end

data/lib/shiba/version.rb

@@ -1,3 +1,3 @@
 module Shiba
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

data/web/results.html.erb

@@ -206,6 +206,20 @@
             return (this.running_cost * 100).toFixed() + "ms";
           else
             return this.running_cost.toFixed(1) + "s";
+        },
+        formatted_result: function() {
+          var rb = this.result_bytes;
+          var result;
+          if ( rb == 0 )
+            return "" + this.result_size + " rows";
+          else if ( rb < 1000 )
+            result = rb + " bytes ";
+          else if ( rb < 1000000 )
+            result = (rb / 1000).toFixed() + "kb ";
+          else
+            result = (rb / 1000000 ).toFixed(1) + "mb ";
+
+          return result + " (" + this.result_size.toLocaleString() + " rows)";
         }
       }
     </script>
@@ -226,7 +240,7 @@
     <script>
       Vue.component('tag-<%= tag %>', {
         template: '#tag-<%= tag %>-template',
-        props: [ 'table_size', 'result_size', 'table', 'cost', 'index', 'join_to', 'index_used', 'running_cost', 'tables', 'rows_read' ],
+        props: [ 'table_size', 'result_size', 'table', 'cost', 'index', 'join_to', 'index_used', 'running_cost', 'tables', 'rows_read', 'result_bytes' ],
         computed: templateComputedFunctions,
         data: function () {
           return { lastRunnningCost: undefined };

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shiba
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Ben Osheroff
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-06 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -121,12 +121,11 @@ files:
 - lib/shiba/activerecord_integration.rb
 - lib/shiba/analyzer.rb
 - lib/shiba/backtrace.rb
-- lib/shiba/checker.rb
 - lib/shiba/configure.rb
 - lib/shiba/connection.rb
 - lib/shiba/connection/mysql.rb
 - lib/shiba/connection/postgres.rb
-- lib/shiba/diff.rb
+- lib/shiba/console.rb
 - lib/shiba/explain.rb
 - lib/shiba/explain/check_support.rb
 - lib/shiba/explain/checks.rb
@@ -139,10 +138,15 @@ files:
 - lib/shiba/index_stats.rb
 - lib/shiba/output.rb
 - lib/shiba/output/tags.yaml
+- lib/shiba/parsers/mysql_select_fields.rb
+- lib/shiba/parsers/shiba_string_scanner.rb
 - lib/shiba/query.rb
 - lib/shiba/query_watcher.rb
 - lib/shiba/review/api.rb
+- lib/shiba/review/cli.rb
 - lib/shiba/review/comment_renderer.rb
+- lib/shiba/review/diff.rb
+- lib/shiba/review/explain_diff.rb
 - lib/shiba/reviewer.rb
 - lib/shiba/setup.rb
 - lib/shiba/table_stats.rb
@@ -177,7 +181,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: A gem that attempts to find bad queries before you shoot self in foot

data/lib/shiba/checker.rb

@@ -1,165 +0,0 @@
-require 'json'
-require 'open3'
-
-require 'shiba/diff'
-require 'shiba/backtrace'
-
-module Shiba
-  # Given an explain log and a diff, returns any explain logs
-  # that appear to be caused by the diff.
-  class Checker
-    Result = Struct.new(:status, :message, :problems)
-
-    attr_reader :options
-
-    def initialize(options)
-      @options = options
-    end
-
-    # Returns a Result object with a status, message, and any problem queries detected.
-    # Query problem format is [ [ "path:lineno", explain ]... ]
-    def run(log)
-      msg = nil
-
-      if options['verbose']
-        puts cmd
-      end
-
-      if changed_files.empty?
-        if options['verbose']
-          msg = "No changes found. Are you sure you specified the correct branch?"
-        end
-        return Result.new(:pass, msg)
-      end
-
-      explains = select_lines_with_changed_files(log)
-      problems = explains.select { |explain| explain["severity"] && explain["severity"] != 'none' }
-
-
-      if options["verbose"]
-        puts problems
-        puts "Updated lines: #{updated_lines}"
-      end
-
-      if problems.empty?
-        msg = "No problems found caused by the diff"
-        return Result.new(:pass, msg)
-      end
-
-      problems.map! do |problem|
-        line = updated_line_from_backtrace(problem["backtrace"], updated_lines)
-        next if line.nil?
-
-        [ line, problem ]
-      end
-      problems.compact!
-
-      if problems.empty?
-        msg = "No problems found caused by the diff"
-        return Result.new(:pass, msg)
-      end
-
-      return Result.new(:fail, "Potential problems", problems)
-    end
-
-    protected
-
-    def updated_line_from_backtrace(backtrace, updates)
-      backtrace.each do |bl|
-        updates.each do |path, lines|
-          next if !bl.start_with?(path)
-          bl =~ /:(\d+):/
-          next if !lines.include?($1.to_i)
-
-          return "#{path}:#{$1}"
-        end
-      end
-
-      return nil
-    end
-
-    def select_lines_with_changed_files(log)
-      patterns = changed_files.map { |path| "-e #{path}" }.join(" ")
-      cmd = "grep #{log} #{patterns}"
-      $stderr.puts cmd if options["verbose"]
-
-      json_lines = `#{cmd}`
-      json_lines.each_line.map { |line| JSON.parse(line) }
-    end
-
-    def changed_files
-      @changed_files ||= begin
-        options['diff'] ? file_diff_names : git_diff_names
-      end
-    end
-
-    def updated_lines
-      return @updated_lines if @updated_lines
-
-
-      out = options['diff'] ? file_diff_lines : git_diff_lines
-      @updated_lines = Shiba::Diff.new(out).updated_lines
-
-
-      @updated_lines.map! do |path, lines|
-        [ Shiba::Backtrace.clean!(path), lines ]
-      end
-    end
-
-    def file_diff_lines
-      File.open(options['diff'])
-    end
-
-    def git_diff_lines
-      run = "git diff#{cmd} --unified=0 --diff-filter=d"
-      if options[:verbose]
-        $stderr.puts run
-      end
-
-      _, out,_,_ = Open3.popen3(run)
-      out
-    end
-
-    # index ade9b24..661d522 100644
-    # --- a/test/app/app.rb
-    # +++ b/test/app/app.rb
-    # @@ -24,4 +24,4 @@ ActiveRecord::Base...
-    # org = Organization.create!(name: 'test')
-    #
-    # file_diff_lines
-    # => test/app/app.rb
-    def file_diff_names
-      file_name_pattern = /^\+\+\+ b\/(.*?)$/
-      f = File.open(options['diff'])
-      f.grep(file_name_pattern) { $1 }
-    end
-
-    def git_diff_names
-      run = "git diff#{cmd} --name-only --diff-filter=d"
-
-      if options[:verbose]
-        $stderr.puts run
-      end
-      result = `#{run}`
-      if $?.exitstatus != 0
-        $stderr.puts result
-        raise Shiba::Error.new "Failed to read changes"
-      end
-
-      result.split("\n")
-    end
-
-    def cmd
-      cmd = case
-      when options["staged"]
-        " --staged"
-      when options["unstaged"]
-        ""
-      else
-        commit = " origin/HEAD"
-        commit << "...#{options["branch"]}" if options["branch"]
-        commit
-      end
-    end
-  end
-end