shiba 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12302c0e4fc56d8c8fc0d14dc6e10a5ae84aeaac2c328937b78b9e14b8122faa
-  data.tar.gz: d0ae437286d3c4975630c5dab8dc5aa109eb38f3ed0f6e606cded20a1201b09f
+  metadata.gz: '0035975f2585607e2aa44f84064fa8ae1bcec4cfbfbf78831595d9de8c690779'
+  data.tar.gz: 3e7bc106fc1c90d3551837411d34f282814f3e8334e215b59dbb67485ff30461
 SHA512:
-  metadata.gz: 92d5a42d2b68381a8f07025dec47e289d75c642a2d53823658966c8032e0e1f2a826f6285b1834747dda798fa25968f27f05c00029f60345303dddc9e2b20c2a
-  data.tar.gz: f4911585fe1a38695b587e11b908265d480523b6e2fcb8966780d80f34ab9612c1a798f612f0a3262f77349d5686a27f4c3da3886824753b36021354c33b267e
+  metadata.gz: c4dff81f128bfc76958f7fc71f26861e5e1594d64c79ec32ea8da8d2421d0a30574e24ad8c2255d38d45f5f0bf6ad30ec2248864b159ab30b6ebcba2c0cb352f
+  data.tar.gz: 4d1f746c8c7d5eedfe9708f4fea54b154c17e79c7c6bdbd91fa41994b6f4f32c1227a4511f9ad04d5a8d56747281835f9a47125a8c5b974366d27dd0ee62f4ce

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    shiba (0.9.2)
+    shiba (0.9.3)
       activesupport
 
 GEM

data/README.md

@@ -8,7 +8,7 @@ Shiba is a tool (currently in alpha) that automatically reviews SQL queries befo
 
 ## Installation
 
-Install using bundler. Note: this gem is not designed to be run on production. It should be required after minitest/rspec.
+Install in a Rails / ActiveRecord project using bundler. Note: this gem is not designed to be run on production. It should be required after minitest/rspec.
 
 ```ruby
 # Gemfile
@@ -39,11 +39,16 @@ SHIBA_DEBUG=true ruby test/controllers/users_controller_test.rb
 # Report available at /tmp/shiba-explain.log-1550099512
 ```
 
+### Postgres Support
+
+Note: Postgres support is under development. For hopefully reliable results, test tables should have at least 1,000 rows.
+
 ## Next steps
 * [Integrate with Github pull requests](#automatic-pull-request-reviews)
 * [Add production stats for realistic analysis](#going-beyond-table-scans)
 * [Preview queries from the developer console](#analyze-queries-from-the-developer-console)
 * [Read more about typical query problems](#typical-query-problems)
+* [Using with languages other than Ruby](#language-support)
 
 
 
@@ -206,3 +211,28 @@ users.size
 Normally a query like this would only become a problem as the app grows in popularity. Fixes include adding `limit` or `find_each`.
 
 With more data, Shiba can help detect this issue when it appears in a pull request.
+
+### Language support
+
+Shiba commands can be used to analyze non Ruby / Rails projects when given a query log file. The log file is a list of queries with the query's backtrace as a SQL comment. The backtrace comment must begin with the word 'shiba' followed by a JSON array of backtrace lines:
+
+```
+SELECT `users`.* FROM `users` WHERE `users`.`email` = 'squirrel@example.com' /*shiba["test/app/app.rb:29:in `<main>'", "Rakefile:0:in `<run>'"]*/
+SELECT  `users`.* FROM `users` ORDER BY `users`.`id` ASC LIMIT 1 /*shiba["test/app/app.rb:30:in `<main>'", "Rakefile:0:in `<run>'"]*/
+```
+
+The generated log file can then be analyzed after installing Ruby.
+
+```console
+gem install bundler
+
+git clone https://github.com/burrito-brothers/shiba.git
+cd shiba
+bundle
+bin/explain -f query.log --database <DB_NAME> --server mysql --json explain.log.json
+bin/review -f explain.log.json
+
+# When no problem queries are found, the command will exit with status 0.
+$ echo $?
+$ 0
+```

data/lib/shiba/activerecord_integration.rb

@@ -74,11 +74,11 @@ module Shiba
 
       # define both minitest and rspec hooks -- it can be
       # unclear in some envs which one is active.  maybe even both could run in one process?  not sure.
-      @shiba_done = false
+      shiba_done = false
       if defined?(Minitest.after_run)
         MiniTest.after_run do
-          yield unless @shiba_done
-          @shiba_done = true
+          yield unless shiba_done
+          shiba_done = true
         end
         @done_hook = :minitest
       end
@@ -86,8 +86,8 @@ module Shiba
       if defined?(RSpec.configure)
         RSpec.configure do |config|
           config.after(:suite) do
-            yield unless @shiba_done
-            @shiba_done = true
+            yield unless shiba_done
+            shiba_done = true
           end
         end
         @done_hook = :rspec

data/lib/shiba/explain/postgres_explain.rb

@@ -17,21 +17,20 @@ module Shiba
 
       def transform_node(node, array)
         case node['Node Type']
-        when "Limit", "LockRows", "Aggregate", "Unique", "Sort", "Hash", "ProjectSet"
+        when "Limit", "LockRows", "Aggregate", "Unique", "Sort", "Hash", "ProjectSet", "Materialize"
           recurse_plans(node, array)
-        when "Nested Loop"
-          with_state(join_type: node["Join Type"]) do
-            recurse_plans(node, array)
-          end
-        when "Hash Join"
-          join_fields = extract_join_key_parts(node['Hash Cond'])
-          with_state(join_fields: join_fields, join_type: "Hash") do
+        when "Hash Join", "Merge Join", "Nested Loop", "BitmapOr"
+          with_state(join_type: node['Node Type']) do
             recurse_plans(node, array)
           end
         when "Bitmap Heap Scan"
           with_state(table: node['Relation Name']) do
             recurse_plans(node, array)
           end
+        when "Subquery Scan"
+          with_state(subquery: true) do
+            recurse_plans(node, array)
+          end
         when "Seq Scan"
           array << {
             "table" => node["Relation Name"],
@@ -48,9 +47,15 @@ module Shiba
             used_key_parts = []
           end
 
+          if @state[:join_type] == 'BitmapOr'
+            access_type = "intersect"
+          else
+            access_type = "ref"
+          end
+
           h = {
             "table" => node["Relation Name"] || @state[:table],
-            "access_type" => "ref",
+            "access_type" => access_type,
             "key" => node["Index Name"],
             "used_key_parts" => used_key_parts
           }
@@ -60,20 +65,36 @@ module Shiba
           end
 
           array << h
+        when "Result"
+          # TBD: What the hell is here?  seems like queries that short-circuit end up here?
+          array << {
+            "extra" => "No tables used"
+          }
         else
+          debugger
           raise "unhandled node: #{node}"
         end
         array
       end
 
       def extract_used_key_parts(cond)
-        conds = Parsers::PostgresExplainIndexConditions.new(cond)
-        conds.fields
+        begin
+          conds = Parsers::PostgresExplainIndexConditions.new(cond)
+          conds.fields
+        rescue Parsers::BadParse => e
+          debugger
+          {}
+        end
       end
 
       def extract_join_key_parts(cond)
-        conds = Parsers::PostgresExplainIndexConditions.new(cond)
-        conds.join_fields
+        begin
+          conds = Parsers::PostgresExplainIndexConditions.new(cond)
+          conds.join_fields
+        rescue Parsers::BadParse => e
+          debugger
+          {}
+        end
       end
 
       def recurse_plans(node, array)

data/lib/shiba/parsers/bad_parse.rb

@@ -0,0 +1,9 @@
+module Shiba
+  module Parsers
+    class BadParse < StandardError
+      def initialize(location, parse_string)
+        super("Parse Error @ '#{location}' while parsing '#{parse_string}'")
+      end
+    end
+  end
+end

data/lib/shiba/parsers/postgres_explain_index_conditions.rb

@@ -1,4 +1,5 @@
 require 'shiba/parsers/shiba_string_scanner'
+require 'shiba/parsers/bad_parse'
 
 module Shiba
   module Parsers
@@ -43,6 +44,7 @@ module Shiba
       def parse_string(sc)
         quote_char = sc.peek(1)
         v = sc.match_quoted_double_escape(quote_char)
+        sc.scan(/::timestamp without time zone/)
         sc.scan(/::\w+(\[\])?/)
         v
       end
@@ -73,13 +75,22 @@ module Shiba
         peek = sc.peek(1)
         if peek == "("
           sc.getch
+          ident = sc.scan(/[^\(\)]+/)
+
           # typed column like (name)::text = 'ben'
-          ident = sc.scan(/[^\)]+/)
           sc.scan(/\)::\S+/)
         elsif peek == '"'
           ident = parse_string(sc)
         else
-          ident = sc.scan(/[^ \.\)\[]+/)
+          ident = sc.scan(/[^ \(\.\)\[]+/)
+
+          # function, lower((name)::text)
+          if sc.scan(/\(/)
+            ident = nil
+            parse_ident(sc)
+            sc.scan(/\)/)
+          end
+
           # field[1] for array fields, not bothering to do brace matching here yet, oy vey
           sc.scan(/\[.*?\]/)
         end
@@ -112,11 +123,15 @@ module Shiba
         table = nil
 
         parse_field(sc)
-        sc.scan(/\s+\S+\s+/) # operator
-        parse_value(sc)
+        sc.scan(/\s+/)
+
+        if !sc.scan(/IS NOT NULL/)
+          sc.scan(/\S+\s+/) # operator
+          parse_value(sc)
+        end
 
         if sc.scan(RPAREN).nil?
-          raise "bad scan; #{sc.inspect}"
+          raise BadParse.new(@sc.peek(15), @string)
         end
       end
     end

data/lib/shiba/version.rb

@@ -1,3 +1,3 @@
 module Shiba
-  VERSION = "0.9.2"
+  VERSION = "0.9.3"
 end

data/shiba.gemspec

@@ -9,8 +9,9 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Ben Osheroff", "Eric Chapweske"]
   spec.email         = ["ben@gimbo.net", "ben.osheroff@gmail.com", "ericis@gmail.com"]
 
-  spec.summary       = %q{A gem that attempts to find bad queries before you shoot self in foot}
-  spec.description   = %q{}
+  spec.summary       = %q{Catch bad SQL queries before they cause problems in production}
+  spec.description   = %q{Use production statistics for realistic SQL query analysis. Finds code that may take down production, including missing indexes, overly broad indexes, and queries that return too much data.
+  }
   spec.homepage      = "https://github.com/burrito-brothers/shiba"
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shiba
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Ben Osheroff
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -81,7 +81,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-description: ''
+description: "Use production statistics for realistic SQL query analysis. Finds code
+  that may take down production, including missing indexes, overly broad indexes,
+  and queries that return too much data.\n  "
 email:
 - ben@gimbo.net
 - ben.osheroff@gmail.com
@@ -139,6 +141,7 @@ files:
 - lib/shiba/index_stats.rb
 - lib/shiba/output.rb
 - lib/shiba/output/tags.yaml
+- lib/shiba/parsers/bad_parse.rb
 - lib/shiba/parsers/mysql_select_fields.rb
 - lib/shiba/parsers/postgres_explain_index_conditions.rb
 - lib/shiba/parsers/shiba_string_scanner.rb
@@ -198,5 +201,5 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: A gem that attempts to find bad queries before you shoot self in foot
+summary: Catch bad SQL queries before they cause problems in production
 test_files: []