shiba 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad898fa0f36457fbd570698c276b567f6f09f1a2
-  data.tar.gz: '028e8a99dff1f8c843bcb222151a0c100f0ebbe1'
+  metadata.gz: 8aaef4cac972cd661d5398bd510d0a78f4d1e078
+  data.tar.gz: 61d077a4f31b4ff21eb652c2c866e8c5f3bb9e49
 SHA512:
-  metadata.gz: 716210a01a6fded73a217b03992371c4d8dcff472f1a69a6195cb53e1dfb122b8c43c512fa7f4ef30bbf776ee8eff3fcabe356bb3e5f4967cec5a4383ec1675a
-  data.tar.gz: 4f7434ec7c78fa7134947ecfe90d9706e4cb22e800eb8036b03216522dee96c8d42e2a33409c5b40e9f4fbb478e48bf6a5e127c3fe0c05284819bd9d87f392ea
+  metadata.gz: 80e2b32747df07efbbd89227b86347530ad955fdc4520f9179bbfe274440397b1063d6dfb50bf2c737f8e88460609e80ae86361b712f9c2e3b0d7ae86d55d728
+  data.tar.gz: 4b540f27e5033c153621a0f2292cba50786857f018d2de54f8d9a5f58755d0a6b8872b2dae5af11e097a98f9b18935499d176ed5aedcedb4a859ce7219c7fc0c

data/.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 .*.sw*
 node_modules
+test/database.yml

data/.travis.yml

@@ -6,8 +6,17 @@ rvm:
 
 services:
  - mysql
+ - postgresql
+
+env:
+  - SHIBA_TEST_ENV=test_postgres
+  - SHIBA_TEST_ENV=test_mysql
 
 before_script:
  - cp .travis/my.cnf ~/.my.cnf
- - mysql -e 'create database shiba_test;'
- - mysql shiba_test -e 'source test/structure.sql'
+
+after_script:
+ - bundle exec shiba review --submit --verbose
+
+addons:
+  postgresql: "9.5"

data/Gemfile

@@ -1,9 +1,8 @@
 source "https://rubygems.org"
 
-gem "mysql2"
 gem "byebug"
 gemspec
 
 group :test do
   gem 'activerecord'
-end
+end

data/Gemfile.lock

@@ -1,8 +1,10 @@
 PATH
   remote: .
   specs:
-    shiba (0.2.3)
+    shiba (0.3.0)
       activesupport
+      mysql2
+      pg
 
 GEM
   remote: https://rubygems.org/
@@ -25,6 +27,7 @@ GEM
       concurrent-ruby (~> 1.0)
     minitest (5.11.3)
     mysql2 (0.5.2)
+    pg (1.1.4)
     rake (10.5.0)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
@@ -37,7 +40,6 @@ DEPENDENCIES
   activerecord
   bundler (~> 2.0)
   byebug
-  mysql2
   rake (~> 10.0)
   shiba!
 

data/README.md

@@ -101,7 +101,7 @@ This information can be obtained by running the bin/dump_stats command in produc
 production$
 git clone https://github.com/burrito-brothers/shiba.git
 cd shiba ; bundle
-bin/dump_stats DATABASE_NAME [MYSQLOPTS] > ~/shiba_index.yml
+bin/mysql_dump_stats -d DATABASE_NAME -h HOST -u USER -pPASS  > ~/shiba_index.yml
 
 local$
 scp production:~/shiba_index.yml RAILS_PROJECT/config

data/bin/explain

@@ -15,58 +15,27 @@ parser = Shiba::Configure.make_options_parser(options)
 parser.banner = "Run a list of queries through shiba's analyzer."
 parser.parse!
 
-option_path = Shiba::Configure.mysql_config_path
-
-if option_path
-  puts "Found config at #{option_path}" if options["verbose"]
-  options['default_file'] ||= option_path
-end
-
-option_file = if options['default_file'] && File.exist?(options['default_file'])
-   File.read(options['default_file'])
-else
-  ""
-end
 
 if options['json'] && options['html']
   $stderr.puts "Can only output to json or html, not both"
   $stderr.puts parser.banner
-  exit 1
-end
-
-if option_file && !options['default_group']
-  if option_file.include?("[client]")
-    options['default_group'] = 'client'
-  end
-  if option_file.include?("[mysql]")
-    options['default_group'] = 'mysql'
-  end
-end
-
-def require_option(parser, name)
-  $stderr.puts "Required: #{name}"
-  $stderr.puts parser.banner
-  exit 1
-end
-
-if !options["username"] && !option_file.include?('user')
-  require_option(parser, 'username')
-end
-
-if !options["database"] && !option_file.include?('database')
-  require_option(parser, 'database')
+  exit 2
 end
 
 file = options.delete("file")
 file = File.open(file, "r") if file
 
-Shiba.configure(options)
+Shiba.configure(options) do |err_msg|
+  $stderr.puts(err_msg)
+  $stderr.puts(parser)
+  exit 2
+end
 
 schema_stats_fname = options["stats"]
 
 if schema_stats_fname && !File.exist?(schema_stats_fname)
   $stderr.puts "No such file: #{schema_stats_fname}"
-  exit 1
+  exit 2
 end
 
 file = $stdin if file.nil?
@@ -88,16 +57,16 @@ if problems.any?
   $stderr.puts "#{problems.size} problematic #{query_word} detected"
 
   if options['json']
-    exit 3
+    exit 1
   end
 
   page = Shiba::Output.new(queries, { 'output' => options['html'] }).make_web!
 
   if !File.exist?(page)
     $stderr.puts("Failed to generate #{page}")
-    exit 1
+    exit 2
   end
 
   $stderr.puts "Report available at #{page}"
-  exit 3
+  exit 1
 end

data/bin/mysql_dump_stats

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'shiba'
+require 'shiba/configure'
+require 'shiba/fuzzer'
+
+options = {}
+parser = Shiba::Configure.make_options_parser(options, only_basics: true)
+parser.banner = "Dump database statistics into yaml file."
+parser.parse!
+
+Shiba.configure(options) do |errmsg|
+  $stderr.puts(errmsg)
+  $stderr.puts(parser.help)
+  exit 1
+end
+
+index = Shiba::Fuzzer.new(Shiba.connection).fetch_index
+puts index.to_yaml

data/bin/postgres_dump_stats

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec `dirname $0`/dump_stats --server postgres $*

data/bin/review

@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+require 'optionparser'
+require 'shiba/reviewer'
+require 'shiba/checker'
+require 'shiba/configure'
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Review changes for query problems. Optionally submit the comments to a Github pull request."
+
+  opts.separator "Required:"
+
+  opts.on("-f","--file FILE", "The explain output log to compare with. Automatically configured when $CI environment variable is set") do |f|
+    options["file"] = f
+  end
+
+  opts.separator ""
+  opts.separator "Git diff options:"
+
+  opts.on("-b", "--branch GIT_BRANCH", "Compare to changes between origin/HEAD and BRANCH. Attempts to read from CI environment when not set.") do |b|
+    options["branch"] = b
+  end
+
+  opts.on("--staged", "Only check files that are staged for commit") do
+    options["staged"] = true
+  end
+
+  opts.on("--unstaged", "Only check files that are not staged for commit") do
+    options["unstaged"] = true
+  end
+
+  opts.separator ""
+  opts.separator "Github options:"
+
+  opts.on("--submit", "Submit comments to Github") do
+    options["submit"] = true
+  end
+
+  opts.on("-p", "--pull-request PR_ID", "The ID of the pull request to comment on. Attempts to read from CI environment when not set.") do |p|
+    options["pull_request"] = p
+  end
+
+  opts.on("-t", "--token TOKEN", "The Github API token to use for commenting. Defaults to $GITHUB_TOKEN.") do |t|
+    options["token"] = t
+  end
+
+  opts.separator ""
+  opts.separator "Common options:"
+
+  opts.on("--verbose", "Verbose/debug mode") do
+    options["verbose"] = true
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.on_tail("--version", "Show version") do
+    require 'shiba/version'
+    puts Shiba::VERSION
+    exit
+  end
+end
+parser.parse!
+
+# This is a noop since it's the default behavior. Ignore.
+if options["staged"] && options["unstaged"]
+  options.delete("staged")
+  options.delete("unstaged")
+end
+
+
+log = options["file"]
+
+if log.nil? && Shiba::Configure.ci?
+  log = options["file"] = File.join(Shiba.path, 'ci.json')
+  $stderr.puts "CI detected, setting file to #{log}" if options["verbose"]
+end
+
+if log.nil?
+  $stderr.puts "Provide an explain log, or run 'shiba explain' to generate one."
+  $stderr.puts ""
+  $stderr.puts parser
+  exit 1
+end
+
+if !File.exist?(log)
+  $stderr.puts "File not found: '#{log}'"
+  exit 1
+end
+
+pr_sha = ENV['TRAVIS_PULL_REQUEST_SHA'] || ENV['CIRCLE_BRANCH']
+
+if options["branch"] == nil && pr_sha && !pr_sha.empty?
+  options["branch"] = pr_sha
+end
+
+if options["token"] == nil
+  options["token"] = ENV['GITHUB_TOKEN']
+end
+
+# https://circleci.com/docs/2.0/env-vars/
+# This may be wrong for circle ci
+pr_id = ENV['TRAVIS_PULL_REQUEST'] || ENV['CIRCLE_PR_NUMBER']
+
+if options["pull_request"] == nil && pr_id && !pr_id.empty?
+  options["pull_request"] = pr_id
+end
+
+if options["verbose"]
+  $stderr.puts "DIFF: #{ENV['DIFF']}" if ENV['DIFF']
+  $stderr.puts "branch: #{options["branch"].inspect}" if options["branch"]
+  $stderr.puts "pull_request: #{options["pull_request"]}" if options["pull_request"]
+end
+
+repo_cmd = "git config --get remote.origin.url"
+repo_url = `#{repo_cmd}`.chomp
+
+if options["verbose"]
+  $stderr.puts "#{repo_cmd}\t#{repo_url}"
+end
+
+def require_option(parser, name)
+  $stderr.puts "Required: #{name}"
+  $stderr.puts ""
+  $stderr.puts parser
+  exit 1
+end
+
+if repo_url.empty?
+  $stderr.puts "'#{Dir.pwd}' does not appear to be a git repo"
+  exit 1
+end
+
+if options["submit"]
+  if (options["branch"].nil? || options["branch"].empty?) && (ENV['DIFF'].nil? || ENV['DIFF'].empty?)
+    require_option(parser, "branch")
+  end
+  require_option(parser, "token") if options["token"].nil?
+  require_option(parser, "pull_request") if options["pull_request"].nil?
+end
+
+if ENV['DIFF']
+  options['diff'] = ENV['DIFF']
+end
+
+# Check to see if the log overlaps with the git diff
+result = Shiba::Checker.new(options).run(log)
+
+if result.message
+  $stderr.puts result.message
+end
+
+if result.status == :pass
+  exit
+end
+
+# Generate comments for the problem queries
+reviewer = Shiba::Reviewer.new(repo_url, result.problems, options)
+
+if !options["submit"] || options["verbose"]
+  reviewer.comments.each do |c|
+    puts "#{c[:path]}:#{c[:line]} (#{c[:position]})"
+    puts c[:body]
+    puts ""
+  end
+end
+
+if options["submit"]
+  if reviewer.repo_host.empty? || reviewer.repo_path.empty?
+    $stderr.puts "Invalid repo url '#{repo_url}' from git config --get remote.origin.url"
+    exit 1
+  end
+
+  reviewer.submit
+end
+
+exit 2

data/bin/shiba

@@ -6,7 +6,7 @@ APP = File.basename(__FILE__)
 
 commands = {
     "explain" => "Generate a report from logged SQL queries",
-    "check"   => "Check staged files for query problems",
+    "review"   => "Review changed files for query problems",
 }
 
 global = OptionParser.new do |opts|
@@ -31,8 +31,8 @@ if command.nil?
 end
 
 if !commands.key?(command)
-    puts "#{APP}: '#{command}' is not a '#{APP}' command. See '#{APP} --help'."
-    exit 1
+  puts "#{APP}: '#{command}' is not a '#{APP}' command. See '#{APP} --help'."
+  exit 2
 end
 
 path = File.join(File.dirname(__FILE__), command)

data/lib/shiba.rb

@@ -1,5 +1,6 @@
 require "shiba/version"
 require "shiba/configure"
+require "shiba/connection"
 require "mysql2"
 require "pp"
 require "byebug" if ENV['SHIBA_DEBUG']
@@ -7,12 +8,46 @@ require "byebug" if ENV['SHIBA_DEBUG']
 module Shiba
   class Error < StandardError; end
 
-  def self.configure(options)
-    @connection_hash = options.select { |k, v| [ 'default_file', 'default_group', 'username', 'database', 'host', 'password'].include?(k) }
+  def self.configure(options, &block)
+    configure_mysql_defaults(options, &block)
+
+    @connection_hash = options.select { |k, v| [ 'default_file', 'default_group', 'server', 'username', 'database', 'host', 'password', 'port'].include?(k) }
     @main_config = Configure.read_config_file(options['config'], "config/shiba.yml")
     @index_config = Configure.read_config_file(options['index'], "config/shiba_index.yml")
   end
 
+  def self.configure_mysql_defaults(options, &block)
+    option_path = Shiba::Configure.mysql_config_path
+
+    if option_path
+      puts "Found config at #{option_path}" if options["verbose"]
+      options['default_file'] ||= option_path
+    end
+
+    option_file = if options['default_file'] && File.exist?(options['default_file'])
+      File.read(options['default_file'])
+    else
+      ""
+    end
+
+    if option_file && !options['default_group']
+      if option_file.include?("[client]")
+        options['default_group'] = 'client'
+      end
+      if option_file.include?("[mysql]")
+        options['default_group'] = 'mysql'
+      end
+    end
+
+    if !options["username"] && !option_file.include?('user')
+      yield('Required: --username')
+    end
+
+    if !options["database"] && !option_file.include?('database')
+      yield('Required: --database')
+    end
+  end
+
   def self.config
     @main_config
   end
@@ -22,16 +57,42 @@ module Shiba
   end
 
   def self.connection
-    @connection ||= Mysql2::Client.new(@connection_hash)
+    return @connection if @connection
+    @connection = Shiba::Connection.build(@connection_hash)
+  end
+
+  def self.database
+    @connection_hash['database']
   end
 
   def self.root
     File.dirname(__dir__)
   end
+
+  def self.path
+    @log_path ||= ENV['SHIBA_PATH'] || try_tmp || use_tmpdir
+  end
+
+  private
+
+  def self.try_tmp
+    return if !Dir.exist?('/tmp')
+    return if !File.writable?('/tmp')
+
+    path = File.join('/tmp', 'shiba')
+    Dir.mkdir(path) if !Dir.exist?(path)
+    path
+  end
+
+  def self.use_tmpdir
+    path = File.join(Dir.tmpdir, 'shiba')
+    Dir.mkdir(path) if !Dir.exist?(path)
+    path
+  end
 end
 
 # This goes at the end so that Shiba.root is defined.
 if defined?(ActiveSupport.on_load)
   require 'shiba/activerecord_integration'
   Shiba::ActiveRecordIntegration.install!
-end
+end