shiba 0.6.4 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: beb49cd39a8eee9adba9a2d1e430c8f8d17c1534fc8a2f609a0e887191bf72cd
-  data.tar.gz: e40ba883aaed19427fcb056c88bd63fe3d8ca1bfd9851575ecba44e54769f546
+SHA1:
+  metadata.gz: f6dbcdad59562d6337cf6357012a6d7e9e08b1c1
+  data.tar.gz: 6583e1f52e1d3ba5f28ed41202b03e9bf6f92124
 SHA512:
-  metadata.gz: 1458a067bbebe0b2910a3fa34cf02fa7666042093ae575ceecc1f771845e17e2b41fff6d9cc760a17103d65ac03718017c827035a9acc2365a10ce671eb410ea
-  data.tar.gz: c173dfa72572b36275f0622641410a06d591a78f5168ddf38ea965d6f859ae250243adbad47ff21bff8c1aaa66ded23043eedb7838208dfc1f372bc30c238e3a
+  metadata.gz: 4612ff6c2cdb56c0ab24ed6bd0c938bbb035de7b881b68cf12ef71fd0507152c26cb60600cd5c7e1a039419a5e83a0af65fdea8f57a0695ecaa88287d079091e
+  data.tar.gz: 3aa4f8e1d74ae4c50e62fd604678ef1b25db1d4007519c9e4708a442039a913bd775d37a49f43db236fc7de44ae906f9b036d0fdd7967704e7113d4cc202e414

data/.gitignore

@@ -9,3 +9,5 @@
 .*.sw*
 node_modules
 test/database.yml
+web/dist/
+logs

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    shiba (0.6.4)
+    shiba (0.8.1)
       activesupport
       mysql2
       pg

data/README.md

@@ -12,7 +12,7 @@ Install using bundler. Note: this gem is not designed to be run on production.
 
 ```ruby
 # Gemfile
-gem 'shiba', :group => :test
+gem 'shiba', :group => :test, :require => 'shiba/setup'
 ```
 
 If your application lazy loads gems, you will to manually require it.
@@ -110,7 +110,7 @@ after_script:
  - bundle exec shiba review --submit
  ```
  
-Add the Github API token you've generated as an environment variable named `GITHUB_TOKEN` at https://travis-ci.com/{organization}/{repo}/settings.
+Add the Github API token you've generated as an environment variable named `SHIBA_GITHUB_TOKEN` at https://travis-ci.com/{organization}/{repo}/settings.
  
 #### CircleCI Integration
  
@@ -123,7 +123,7 @@ To integrate with CircleCI, add this after the the test run step in `.circleci/c
     command: bundle exec shiba review --submit
 ```
 
-An environment variable named `GITHUB_TOKEN` will need to be configured on CircleCI under *Project settings > Environment Variables*
+An environment variable named `SHIBA_GITHUB_TOKEN` will need to be configured on CircleCI under *Project settings > Environment Variables*
 
 #### Custom CI Integration
 

data/Rakefile

@@ -2,10 +2,12 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/*_test.rb']
-  t.verbose = true
-end
+    t.libs << "test"
+    t.test_files = FileList['test/*_test.rb']
+    t.verbose = true
+  end
+
+task :default => :test
 
 task :build_web do
   Dir.chdir(File.join(File.dirname(__FILE__), "web"))
@@ -13,6 +15,14 @@ task :build_web do
   sh("npm run build")
 end
 
+task :check_master do
+  current_branch = `git rev-parse --abbrev-ref HEAD`.chomp
+  if "master" != current_branch
+    $stderr.puts "\n===== Warning: Not on master. running on branch #{current_branch} =====\n\n"
+  end
+end
+
+Rake::Task[:release].prerequisites.unshift(:check_master)
 Rake::Task[:release].prerequisites.unshift(:build_web)
 
-task :default => :test
+task :default => :test

data/bin/explain

@@ -7,8 +7,6 @@ require 'shiba/table_stats'
 require 'shiba/configure'
 require 'shiba/output'
 
-require 'optionparser'
-
 options = {}
 
 parser = Shiba::Configure.make_options_parser(options)

data/bin/review

@@ -1,71 +1,7 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
-require 'optionparser'
-require 'shiba/configure'
-require 'shiba/reviewer'
-require 'shiba/review/explain_diff'
 require 'shiba/review/cli'
-require 'json'
 
 cli = Shiba::Review::CLI.new
-cli.report_options("diff", "branch", "pull_request")
-
-if !cli.valid?
-  $stderr.puts cli.failure
-  exit 1
-end
-
-explain_diff = Shiba::Review::ExplainDiff.new(cli.options["file"], cli.options)
-
-problems = if explain_diff.diff_requested_by_user?
-  result = explain_diff.result
-
-  if result.message
-    $stderr.puts result.message
-  end
-
-  if result.status == :pass
-    exit
-  end
-
-  explain_diff.problems
-else
-  explains = File.open(cli.options["file"]).each_line.map { |json| JSON.parse(json) }
-  bad = explains.select { |explain| explain["severity"] && explain["severity"] != 'none' }
-  bad.map { |explain| [ "#{explain["sql"]}:-2", explain ] }
-end
-
-repo_cmd = "git config --get remote.origin.url"
-repo_url = `#{repo_cmd}`.chomp
-
-if cli.options["verbose"]
-  $stderr.puts "#{repo_cmd}\t#{repo_url}"
-end
-
-if repo_url.empty?
-  $stderr.puts "'#{Dir.pwd}' does not appear to be a git repo"
-  exit 1
-end
-
-# Generate comments for the problem queries
-reviewer = Shiba::Reviewer.new(repo_url, problems, cli.options)
-
-if !cli.options["submit"] || cli.options["verbose"]
-  reviewer.comments.each do |c|
-    puts "#{c[:path]}:#{c[:line]} (#{c[:position]})"
-    puts c[:body]
-    puts ""
-  end
-end
-
-if cli.options["submit"]
-  if reviewer.repo_host.empty? || reviewer.repo_path.empty?
-    $stderr.puts "Invalid repo url '#{repo_url}' from git config --get remote.origin.url"
-    exit 1
-  end
-
-  reviewer.submit
-end
-
-exit 2
+exit cli.run

data/bin/shiba

@@ -7,7 +7,8 @@ APP = File.basename(__FILE__)
 commands = {
     "explain" => "Generate a report from logged SQL queries",
     "review"  => "Review changed files for query problems",
-    "dump_stats" => "Collect database statistics for more accurate analysis"
+    "dump_stats" => "Collect database statistics for more accurate analysis",
+    "web" => "Generate a report from JSON-explain"
 }
 
 global = OptionParser.new do |opts|
@@ -38,4 +39,4 @@ end
 
 path = File.join(File.dirname(__FILE__), command)
 
-Kernel.exec(path, *ARGV)
+Kernel.exec(path, *ARGV)

data/bin/web

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'shiba/output'
+require 'json'
+require 'optionparser'
+require 'set'
+
+options = {}
+
+parser = OptionParser.new do |opts|
+  opts.on("-h", "--html FILE", "write html report here.") do |h|
+    options["html"] = h
+  end
+end
+
+parser.banner = "Shiba web: accept JSON-explain on stdin, write out an HTML report"
+parser.parse!
+
+if !options['html']
+  $stderr.puts parser.help
+  $stderr.puts "required option: --html"
+  exit 2
+end
+
+queries = []
+query_md5s = Set.new
+
+while line = $stdin.gets
+  query = JSON.parse(line)
+  next if query_md5s.include?(query['md5'])
+  queries << JSON.parse(line)
+  query_md5s << query['md5']
+end
+
+page = Shiba::Output.new(queries, { 'output' => options['html'] }).make_web!
+if !File.exist?(page)
+  $stderr.puts("Failed to generate #{page}")
+  exit 2
+end
+
+$stderr.puts "Report available at #{page}"
+

data/lib/shiba.rb

@@ -75,7 +75,7 @@ module Shiba
   end
 
   def self.path
-    @log_path ||= ENV['SHIBA_PATH'] || try_tmp || use_tmpdir
+    @log_path ||= ENV['SHIBA_DIR'] || try_tmp || use_tmpdir
   end
 
   private
@@ -94,10 +94,4 @@ module Shiba
     Dir.mkdir(path) if !Dir.exist?(path)
     path
   end
-end
-
-# This goes at the end so that Shiba.root is defined.
-if defined?(ActiveSupport.on_load)
-  require 'shiba/activerecord_integration'
-  Shiba::ActiveRecordIntegration.install!
-end
+end

data/lib/shiba/activerecord_integration.rb

@@ -7,13 +7,17 @@ module Shiba
   # Integrates ActiveRecord with the Query Watcher by setting up the query log path, and the
   # connection options for the explain command, which it runs when the process exits.
   #
-  # SHIBA_OUT and SHIBA_DEBUG=true environment variables may be set.
+  # SHIBA_DIR, SHIBA_QUERY_LOG_NAME and SHIBA_DEBUG=true environment variables may be set.
   class ActiveRecordIntegration
 
     attr_reader :path, :watcher
 
     def self.install!
       return false if @installed
+      if defined?(Rails.env) && Rails.env.production?
+        Rails.logger.error("Shiba watcher is not intended to run in production, stopping install.")
+        return false
+      end
 
       ActiveSupport.on_load(:active_record) do
         Shiba::ActiveRecordIntegration.start_watcher
@@ -32,7 +36,7 @@ module Shiba
           c = { host: cx.host, database: cx.db, username: cx.user, password: cx.pass, port: cx.port, server: 'postgres' }
         end
 
-        options = {
+        {
           'host' =>     c[:host],
           'database' => c[:database],
           'user' =>     c[:username],
@@ -61,17 +65,16 @@ module Shiba
     end
 
     def self.log_path
-      name = ENV["SHIBA_OUT"] || "query.log-#{Time.now.to_i}"
+      name = ENV["SHIBA_QUERY_LOG_NAME"] || "query.log-#{Time.now.to_i}"
       File.join(Shiba.path, name)
     end
 
     def self.run_explain(file, path)
       file.close
-      puts ""
 
       cmd = "shiba explain #{database_args} --file #{path}"
-      if ENV['SHIBA_OUT']
-        cmd << " --json #{File.join(Shiba.path, "#{ENV["SHIBA_OUT"]}.json")}"
+      if ENV['SHIBA_QUERY_LOG_NAME']
+        cmd << " --json #{File.join(Shiba.path, "#{ENV["SHIBA_QUERY_LOG_NAME"]}.json")}"
       elsif Shiba::Configure.ci?
         cmd << " --json #{File.join(Shiba.path, 'ci.json')}"
       end

data/lib/shiba/explain/checks.rb

@@ -129,6 +129,12 @@ module Shiba
         end
 
         @cost = Shiba::Explain::COST_PER_ROW_READ * rows_read
+
+        # pin fully missed indexes to a 'low' threshold
+        if @access_type == 'access_type_tablescan'
+          @cost = [0.01, @cost].max
+        end
+
         @result.cost += @cost
 
         @tbl_message['cost'] = @cost

data/lib/shiba/output.rb

@@ -61,7 +61,11 @@ module Shiba
     def make_web!
       data = as_json
 
-      index = File.read(File.join(WEB_PATH, "dist", "index.html"))
+      index_path = File.join(WEB_PATH, "dist", "index.html")
+      if !File.exist?(index_path)
+        raise Shiba::Error.new("dist/index.html not found. Try running 'rake build_web'")
+      end
+      index = File.read(index_path)
       data_block = "var shibaData = #{JSON.dump(as_json)};\n"
 
       index.sub!(%r{<script src=(.*?)>}) do |match|

data/lib/shiba/parsers/mysql_select_fields.rb

@@ -52,7 +52,10 @@ module Shiba
             tables[table] << col
           elsif sc.scan(/(\d+|NULL|'.*?') AS `(.*?)`/m)
           else
-            raise "unknown stuff: in #{@sql}: #{@sc.rest}"
+            if ENV['SHIBA_DEBUG']
+              raise Shiba::Error.new("unknown stuff: in #{@sql}: #{@sc.rest}")
+            end
+            return {}
           end
 
           sc.scan(/,/)

data/lib/shiba/review/cli.rb

@@ -1,3 +1,9 @@
+require 'optionparser'
+require 'shiba/configure'
+require 'shiba/reviewer'
+require 'shiba/review/explain_diff'
+require 'json'
+
 module Shiba
   module Review
     # Builds options for interacting with the reviewer via the command line.
@@ -18,13 +24,108 @@ module Shiba
     # => "An error message with command line help."
     class CLI
 
-      attr_reader :errors
+      attr_reader :out, :err, :input
 
-      def initialize
-        @user_options = {}
+      # Options may be provided for testing, in which case the option parser is skipped.
+      # When this happens, default options are also skipped.
+      def initialize(out: $stdout, err: $stderr, input: $stdin, options: nil)
+        @out = out
+        @err = err
+        @input = input
+        @user_options = options || {}
         @errors = []
-        parser.parse!
-        @options = default_options.merge(@user_options)
+        parser.parse! if options.nil?
+        @options = options || default_options.merge(@user_options)
+      end
+
+      # Generates the review, returning an exit status code.
+      # Prints to @out / @err, which default to STDOUT/STDERR.
+      def run
+        report_options("diff", "branch", "pull_request")
+
+        if !valid?
+          err.puts failure
+          return 1
+        end
+
+        explain_diff = Shiba::Review::ExplainDiff.new(options["file"], options)
+
+        problems = if explain_diff.diff_requested_by_user?
+          result = explain_diff.result
+
+          if result.message
+            @err.puts result.message
+          end
+
+          if result.status == :pass
+            return 0
+          end
+
+          explain_diff.problems
+        else
+          # Find all problem explains
+          begin
+            explains = explain_file.each_line.map { |json| JSON.parse(json) }
+            bad = explains.select { |explain| explain["severity"] && explain["severity"] != 'none' }
+            bad.map { |explain| [ "#{explain["sql"]}:-2", explain ] }
+          rescue Interrupt
+            @err.puts "SIGINT: Canceled reading from STDIN. To read from an explain log, provide the --file option."
+            exit 1
+          end
+        end
+
+        if problems.empty?
+          return 0
+        end
+
+        # Dedup
+        problems.uniq! { |_,p| p["md5"] }
+
+        # Output problem explains, this can be provided as a file to shiba review for comments.
+        if options["raw"]
+          pr = options["pull_request"]
+          if pr
+            problems.each { |_,problem| problem["pull_request"] = pr }
+          end
+
+
+          problems.each { |_,problem| @out.puts JSON.dump(problem) }
+          return 2
+        end
+
+        # Generate comments for the problem queries
+        repo_cmd = "git config --get remote.origin.url"
+        repo_url = `#{repo_cmd}`.chomp
+
+        if options["verbose"]
+          @err.puts "#{repo_cmd}\t#{repo_url}"
+        end
+
+        if repo_url.empty?
+          @err.puts "'#{Dir.pwd}' does not appear to be a git repo"
+          return 1
+        end
+
+        reviewer = Shiba::Reviewer.new(repo_url, problems, options)
+
+        if !options["submit"] || options["verbose"]
+          reviewer.comments.each do |c|
+            @out.puts "#{c[:path]}:#{c[:line]} (#{c[:position]})"
+            @out.puts c[:body]
+            @out.puts ""
+          end
+        end
+
+        if options["submit"]
+          if reviewer.repo_host.empty? || reviewer.repo_path.empty?
+            @err.puts "Invalid repo url '#{repo_url}' from git config --get remote.origin.url"
+            return 1
+          end
+
+          reviewer.submit
+        end
+
+        return 2
       end
 
       def options
@@ -36,14 +137,12 @@ module Shiba
 
         validate_log_path
         #validate_git_repo if branch || options["submit"]
-        description = "Provide an explain log, or run 'shiba explain' to generate one."
-
-        require_option("file", description: description)
 
         if options["submit"]
           require_option("branch") if options["diff"].nil?
-          require_option("token")
+          require_option("token", description: "This can be read from the $SHIBA_GITHUB_TOKEN environment variable.")
           require_option("pull_request")
+          error("Must specify either 'submit' or 'raw' output option, not both") if options["raw"]
         end
 
         @errors.empty?
@@ -69,16 +168,25 @@ module Shiba
 
       protected
 
+      def explain_file
+        options.key?('file') ? File.open(options['file']) : @input
+      end
+
       def parser
         @parser ||= OptionParser.new do |opts|
-          opts.banner = "Review changes for query problems. Optionally submit the comments to a Github pull request."
+          opts.banner = "Reads from a file or stdin to review changes for query problems. Optionally submit the comments to a Github pull request."
 
-          opts.separator "Required:"
+          opts.separator ""
+          opts.separator "IO options:"
 
-          opts.on("-f","--file FILE", "The explain output log to compare with. Automatically configured when $CI environment variable is set") do |f|
+          opts.on("-f","--file FILE", "The JSON explain log to compare with. Automatically configured when $CI environment variable is set") do |f|
             @user_options["file"] = f
           end
 
+          opts.on("--raw", "Print the raw JSON with the pull request id") do |r|
+            @user_options["raw"] = r
+          end
+
           opts.separator ""
           opts.separator "Git diff options:"
 
@@ -105,7 +213,7 @@ module Shiba
             @user_options["pull_request"] = p
           end
 
-          opts.on("-t", "--token TOKEN", "The Github API token to use for commenting. Defaults to $GITHUB_TOKEN.") do |t|
+          opts.on("-t", "--token TOKEN", "The Github API token to use for commenting. Defaults to $SHIBA_GITHUB_TOKEN.") do |t|
             @user_options["token"] = t
           end
 
@@ -117,13 +225,13 @@ module Shiba
           end
 
           opts.on_tail("-h", "--help", "Show this message") do
-            puts opts
+            @out.puts opts
             exit
           end
 
           opts.on_tail("--version", "Show version") do
             require 'shiba/version'
-            puts Shiba::VERSION
+            @out.puts Shiba::VERSION
             exit
           end
         end
@@ -139,12 +247,12 @@ module Shiba
         if Shiba::Configure.ci?
           report("Finding default options from CI environment.")
 
-          defaults["file"]         = ci_explain_log_path
-          defaults["pull_request"] = ci_pull_request
-          defaults["branch"]       = ci_branch if !defaults['diff']
+          defaults["file"]         = ci_explain_log_path if ci_explain_log_path
+          defaults["pull_request"] = ci_pull_request     if ci_pull_request
+          defaults["branch"]       = ci_branch           if !defaults['diff'] && ci_branch
         end
 
-        defaults["token"] = ENV['GITHUB_TOKEN'] if ENV['GITHUB_TOKEN']
+        defaults["token"] = ENV['SHIBA_GITHUB_TOKEN'] if ENV['SHIBA_GITHUB_TOKEN']
 
         defaults
       end
@@ -157,7 +265,8 @@ module Shiba
       end
 
       def ci_explain_log_path
-        File.join(Shiba.path, 'ci.json')
+        name = ENV['SHIBA_QUERY_LOG_NAME'] || 'ci'
+        File.join(Shiba.path, "#{name}.json")
       end
 
       def ci_branch
@@ -184,7 +293,7 @@ module Shiba
       end
 
       def report(message)
-        $stderr.puts message if @user_options["verbose"]
+        @err.puts message if @user_options["verbose"]
       end
 
       def error(message, help: false)