shiba 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cd7eeaf515680a21faf4b12533d160db2b86a89
-  data.tar.gz: c44357f6abd8e38ec6c8df9139e81826b39aabcf
+  metadata.gz: 1976edef35324cc1d8c62239938ef8fa2ceaa247
+  data.tar.gz: 26425a11975b434ca85f100e1608ac79812b2b8d
 SHA512:
-  metadata.gz: 46d865b67f759ed0e694a9c88aaaf4dbbc932c006b2f0ea088a715d55d07d45409d441b0b518ebd10cc7cf9b5eb3dc861437f8473f1eb293c443e54373476f2b
-  data.tar.gz: 6a5bba249b95220a9fbd8bf1f84d4c04a4a8ba74ec500ddc826994f7b2b07c7d23ef67d6e0ad9cbda9aaebc99983eb81eac3a656688bdb612bd4505168e653e4
+  metadata.gz: 728df84438f0eb7d40dc3c05a718ee913dafc6a7a9275859f489875e91ccf6ba73ddb5431eb61650f7e4ddf8be9b39f3f52dbfec3a8f45bfe8b893429bb77fdb
+  data.tar.gz: 113296428f8336a3a21e639415badf893bd0ed178c17b536aa282e4575ecd6e6b2dc385bae336ca88ce5ceafe216e3da2c2943cb615b969294d0e449779947a7

data/Gemfile

@@ -3,3 +3,7 @@ source "https://rubygems.org"
 gem "mysql2"
 gem "byebug"
 gemspec
+
+group :test do
+  gem 'activerecord'
+end

data/Gemfile.lock

@@ -1,17 +1,24 @@
 PATH
   remote: .
   specs:
-    shiba (0.2.0)
+    shiba (0.2.2)
       activesupport
 
 GEM
   remote: https://rubygems.org/
   specs:
+    activemodel (5.2.2)
+      activesupport (= 5.2.2)
+    activerecord (5.2.2)
+      activemodel (= 5.2.2)
+      activesupport (= 5.2.2)
+      arel (>= 9.0)
     activesupport (5.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+    arel (9.0.0)
     byebug (10.0.2)
     concurrent-ruby (1.1.4)
     i18n (1.5.3)
@@ -27,6 +34,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord
   bundler (~> 2.0)
   byebug
   mysql2

data/README.md

@@ -5,11 +5,9 @@
 Shiba is a tool that helps catch poorly performing queries before they cause problems in production, including:
 
 * Full table scans
-* Non selective indexes
+* Poorly performing indexes
 
-By default, it will pretty much only detect queries that miss indexes. As it's fed more information, it warns about advanced problems, such as queries that use indexes but are still very expensive.
-
-To help find such queries, Shiba monitors test runs for ActiveRecord queries. A warning and report are then generated. Shiba is further capable of only warning on changes that occured on a particular git branch/pull request to allow for CI integration.
+By default, it will pretty much only detect queries that miss indexes. As it's fed more information, it warns about advanced problems, such as queries that use indexes but are still very expensive. To help find such queries, Shiba monitors test runs for ActiveRecord queries. A warning and report are then generated
 
 ## Installation
 
@@ -19,8 +17,17 @@ Install using bundler. Note: this gem is not designed to be run on production.
 gem 'shiba', :group => :test, :require => true
 ```
 
+If this doesn't magically work, you can manually configure it using an initializer:
+
+```ruby
+  require 'shiba/activerecord_integration'
+  Shiba::ActiveRecordIntegration.install!
+```
+
 ## Usage
 
+A report will only be generated when problem queries are detected.
+
 ```ruby
 # Install
 bundle
@@ -83,7 +90,7 @@ For smarter analysis, Shiba requires general statistics about production data, s
 This information can be obtained by running the bin/dump_stats command in production.
 
 ```console
-production$ 
+production$
 git clone https://github.com/burrito-brothers/shiba.git
 cd shiba ; bundle
 bin/dump_stats DATABASE_NAME [MYSQLOPTS] > ~/shiba_index.yml
@@ -102,18 +109,18 @@ users:
       name: PRIMARY
       columns:
       - column: id
-        rows_per: 1
+        rows_per: 1 # one row per unique `id`
       unique: true
-    index_users_on_login:
-      name: index_users_on_login
+    index_users_on_email:
+      name: index_users_on_email
       columns:
-      - column: login
-        rows_per: 1
+      - column: email
+        rows_per: 1 # one row per email address (also unique)
       unique: true
-    index_users_on_created_by_id:
-      name: index_users_on_created_by_id
+    index_users_on_organization_id:
+      name: index_users_on_organization_id
       columns:
-      - column: created_by_id
-        rows_per: 3
+      - column: organization_id
+        rows_per: 20% # each organization has, on average, 20% or 2000 users.
       unique: false
 ```

data/bin/dump_stats

@@ -10,7 +10,7 @@ esac
 DATABASE=$1
 if [ -z "$DATABASE" ]
 then
-  echo "usage: dump_stats -tables [table1,table2...] DATABASE [ ...mysql_args]"
+  echo "usage: dump_stats [-tables table1,table2...] DATABASE [ ...mysql_args]" >&2
   exit 1
 fi
 
@@ -23,6 +23,12 @@ shift
 MYSQL_STATS=`mktemp`
 mysql $* -ABe "select * from information_schema.statistics where table_schema = '$DATABASE'" > $MYSQL_STATS
 
+if ! [ -s $MYSQL_STATS ]
+then
+  echo "no such database: $DATABASE" >&2
+  exit 1
+fi
+
 if [ "$TABLES" ]
 then
   filtered=`mktemp`
@@ -35,4 +41,4 @@ then
   MYSQL_STATS=$filtered
 fi
 
-bundle exec ruby -e "require 'shiba/index'; puts Shiba::Index.parse('$MYSQL_STATS').to_yaml"
+bundle exec ruby -e "require 'shiba/index'; puts Shiba::Index.parse('$MYSQL_STATS').to_yaml"

data/lib/shiba.rb

@@ -31,4 +31,7 @@ module Shiba
 end
 
 # This goes at the end so that Shiba.root is defined.
-require "shiba/railtie" if defined?(Rails)
+if defined?(ActiveSupport.on_load)
+  require 'shiba/activerecord_integration'
+  Shiba::ActiveRecordIntegration.install!
+end

data/lib/shiba/activerecord_integration.rb

@@ -0,0 +1,71 @@
+require 'shiba/query_watcher'
+require 'active_support/notifications'
+require 'active_support/lazy_load_hooks'
+
+module Shiba
+  # Integrates ActiveRecord with the Query Watcher by setting up the query log path, and the
+  # connection options for the explain command, which it runs when the process exits.
+  #
+  # SHIBA_OUT=<log path> and SHIBA_DEBUG=true environment variables may be set.
+  class ActiveRecordIntegration
+
+    attr_reader :path, :watcher
+
+    def self.install!
+      return false if @installed
+
+      ActiveSupport.on_load(:active_record) do
+        Shiba::ActiveRecordIntegration.start_watcher
+      end
+
+      @installed = true
+    end
+
+    protected
+
+    def self.start_watcher
+      if ENV['SHIBA_DEBUG']
+        $stderr.puts("starting shiba watcher")
+      end
+
+      path = ENV['SHIBA_OUT'] || make_tmp_path
+
+      file = File.open(path, 'a')
+      watcher = QueryWatcher.new(file)
+
+      ActiveSupport::Notifications.subscribe('sql.active_record', watcher)
+      at_exit { run_explain(file, path) }
+    rescue => e
+      $stderr.puts("Shiba failed to load")
+      $stderr.puts(e.message, e.backtrace.join("\n"))
+    end
+
+    def self.make_tmp_path
+      "/tmp/shiba-query.log-#{Time.now.to_i}"
+    end
+
+    def self.run_explain(file, path)
+      file.close
+      puts ""
+      cmd = "shiba explain #{database_args} --file #{path}"
+      if ENV['SHIBA_DEBUG']
+        $stderr.puts("running:")
+        $stderr.puts(cmd)
+      end
+      system(cmd)
+    end
+
+    def self.database_args
+      c = ActiveRecord::Base.connection.raw_connection.query_options
+      options = {
+      'host':     c[:host],
+      'database': c[:database],
+      'user':     c[:username],
+      'password': c[:password]
+      }
+
+      options.reject { |k,v| v.nil? }.map { |k,v| "--#{k} #{v}" }.join(" ")
+    end
+
+  end
+end

data/lib/shiba/backtrace.rb

@@ -2,11 +2,15 @@ require 'open3'
 
 module Shiba
   module Backtrace
-    IGNORE = /\.rvm|gem|vendor\/|rbenv|seed|db|shiba|test|spec/
+
+    def self.ignore
+      @ignore ||= [ '.rvm', 'gem', 'vendor', 'rbenv', 'seed',
+        'db', 'test', 'spec', 'lib/shiba' ]
+    end
 
     # 8 backtrace lines starting from the app caller, cleaned of app/project cruft.
     def self.from_app
-      app_line_idx = caller_locations.index { |line| line.to_s !~ IGNORE }
+      app_line_idx = caller_locations.index { |line| line.to_s !~ ignore_pattern }
       if app_line_idx == nil
         return
       end
@@ -17,14 +21,18 @@ module Shiba
     end
 
     def self.clean!(line)
-      line.sub!(backtrace_ignore_pattern, '')
+      line.sub!(backtrace_clean_pattern, '')
       line
     end
 
     protected
 
-    def self.backtrace_ignore_pattern
-      @roots ||= begin
+    def self.ignore_pattern
+      @pattern ||= Regexp.new(ignore.map { |word| Regexp.escape(word) }.join("|"))
+    end
+
+    def self.backtrace_clean_pattern
+      @backtrace_clean_pattern ||= begin
         paths = Gem.path
         paths << Rails.root.to_s if defined?(Rails.root)
         paths << repo_root

data/lib/shiba/index_stats.rb

@@ -49,6 +49,8 @@ module Shiba
       def encode_with(coder)
         coder.map = self.to_h.stringify_keys
         coder.map.delete('table')
+        coder.map.delete('unique') unless unique
+
         coder.tag = nil
       end
     end

data/lib/shiba/query_watcher.rb

@@ -1,15 +1,10 @@
 require 'shiba/query'
 require 'shiba/backtrace'
-require 'json'
-require 'rails'
 
 module Shiba
+  # Logs ActiveRecord SELECT queries that originate from application code.
   class QueryWatcher
 
-    def self.watch(file)
-      new(file).tap { |w| w.watch }
-    end
-
     attr_reader :queries
 
     def initialize(file)
@@ -18,22 +13,19 @@ module Shiba
       @queries = {}
     end
 
-    # Logs ActiveRecord SELECT queries that originate from application code.
-    def watch
-      ActiveSupport::Notifications.subscribe('sql.active_record') do |name, start, finish, id, payload|
-        sql = payload[:sql]
-
-        if sql.start_with?("SELECT")
-          fingerprint = Query.get_fingerprint(sql)
-          if !@queries[fingerprint]
-            if lines = Backtrace.from_app
-              @file.puts("#{sql} /*shiba#{lines}*/")
-            end
-          end
-          @queries[fingerprint] = true
-        end
-      end
+    def call(name, start, finish, id, payload)
+      sql = payload[:sql]
+      return if !sql.start_with?("SELECT")
+
+      fingerprint = Query.get_fingerprint(sql)
+      return if @queries[fingerprint]
+
+      lines = Backtrace.from_app
+      return if !lines
+
+      @file.puts("#{sql} /*shiba#{lines}*/")
+      @queries[fingerprint] = true
     end
 
   end
-end
+end

data/lib/shiba/version.rb

@@ -1,3 +1,3 @@
 module Shiba
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shiba
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Ben Osheroff
@@ -88,6 +88,7 @@ files:
 - cmd/inspect.go
 - cmd/parse.go
 - lib/shiba.rb
+- lib/shiba/activerecord_integration.rb
 - lib/shiba/analyzer.rb
 - lib/shiba/backtrace.rb
 - lib/shiba/checker.rb
@@ -101,7 +102,6 @@ files:
 - lib/shiba/output/tags.yaml
 - lib/shiba/query.rb
 - lib/shiba/query_watcher.rb
-- lib/shiba/railtie.rb
 - lib/shiba/table_stats.rb
 - lib/shiba/version.rb
 - shiba.gemspec
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: A gem that attempts to find bad queries before you shoot self in foot

data/lib/shiba/railtie.rb

@@ -1,43 +0,0 @@
-require 'shiba/query_watcher'
-
-class Shiba::Railtie < Rails::Railtie
-  # Logging is enabled when:
-  #  1. SHIBA_OUT environment variable is set to an existing file path.
-  #  2. RSpec/MiniTest exists, in which case a fallback query log is generated at /tmp
-  config.after_initialize do
-    begin
-      path = ENV['SHIBA_OUT'] || "/tmp/shiba-query.log-#{Time.now.to_i}"
-      f = File.open(path, 'a')
-      watcher = Shiba::QueryWatcher.watch(f)
-      next if watcher.nil?
-
-      at_exit do
-        f.close
-        puts ""
-        cmd = "shiba explain #{database_args} --file #{path}"
-        if ENV['SHIBA_DEBUG']
-          $stderr.puts("running:")
-          $stderr.puts(cmd)
-        end
-        system(cmd)
-      end
-
-    rescue => e
-      $stderr.puts("Shiba failed to load")
-      $stderr.puts(e.message, e.backtrace.join("\n"))
-    end
-  end
-
-  def self.database_args
-    c = ActiveRecord::Base.configurations['test']
-    options = {
-      'host':     c['host'],
-      'database': c['database'],
-      'user':     c['username'],
-      'password': c['password']
-    }
-
-    options.reject { |k,v| v.nil? }.map { |k,v| "--#{k} #{v}" }.join(" ")
-  end
-
-end