shiba 0.2.3 → 0.3.0

data/lib/shiba/diff.rb

@@ -12,23 +12,28 @@ module Shiba
     # The position value equals the number of lines down from the first "@@" hunk header
     # in the file you want to add a comment.
 
-    # diff = `git diff --unified=0`
-    # parse_diff(StringIO.new(diff))
-    # => "hello.rb:1"
-    # => "hello.rb:2"
-    # => "test.rb:5"
-
-  # For simplicity, the default output of git diff is not supported.
-  # The expected format is from 'git diff unified=0'
-
   attr_reader :status
 
   def initialize(file)
+    # Fixme. seems like enumerables should work in general.
+    if !file.respond_to?(:pos)
+      raise StandardError.new("Diff file does not appear to be a seekable IO object.")
+    end
     @diff = file
     @status = :new
   end
 
   # Returns the file and line numbers that contain inserts. Deletions are ignored.
+  # For simplicity, the default output of git diff is not supported.
+  # The expected format is from 'git diff unified=0'
+  #
+  # Example:
+  # diff = `git diff --unified=0`
+  # Diff.new(StringIO.new(diff))
+  # => [ [ "hello.rb", 1..3 ]
+  # =>   [ "hello.rb", 7..7 ]
+  # =>   [ "test.rb", 23..23 ]
+  # => ]
   def updated_lines
     io = @diff.each_line
     path = nil
@@ -53,6 +58,11 @@ module Shiba
   # Returns the position in the diff, after the relevant file header,
   # that contains the specified file/lineno modification.
   # Only supports finding the position in the destination / newest version of the file.
+  #
+  # Example:
+  # diff = Diff.new(`git diff`)
+  # diff.find_position("test.rb", 3)
+  # => 5
   def find_position(path, line_number)
     io = @diff.each_line # maybe redundant?
 
@@ -101,11 +111,11 @@ module Shiba
     end
 
     def file_header?(line)
-      line.match?(FILE_PATTERN)
+      line =~ FILE_PATTERN
     end
 
     def hunk_header?(line)
-      line.match?(LINE_PATTERN)
+      LINE_PATTERN =~ line
     end
 
     def line_numbers_for_destination(diff_line)

data/lib/shiba/explain.rb

@@ -1,5 +1,7 @@
 require 'json'
 require 'shiba/index'
+require 'shiba/explain/mysql_explain'
+require 'shiba/explain/postgres_explain'
 
 module Shiba
   class Explain
@@ -12,9 +14,13 @@ module Shiba
       end
 
       @options = options
-      ex = Shiba.connection.query("EXPLAIN FORMAT=JSON #{@sql}").to_a
-      @explain_json = JSON.parse(ex.first['EXPLAIN'])
-      @rows = self.class.transform_json(@explain_json['query_block'])
+      @explain_json = Shiba.connection.explain(@sql)
+
+      if Shiba.connection.mysql?
+        @rows = Shiba::Explain::MysqlExplain.new.transform_json(@explain_json['query_block'])
+      else
+        @rows = Shiba::Explain::PostgresExplain.new(@explain_json).transform
+      end
       @stats = stats
       run_checks!
     end
@@ -47,48 +53,6 @@ module Shiba
       table
     end
 
-    def self.transform_table(table, extra = {})
-      t = table
-      res = {}
-      res['table'] = t['table_name']
-      res['access_type'] = t['access_type']
-      res['key'] = t['key']
-      res['used_key_parts'] = t['used_key_parts'] if t['used_key_parts']
-      res['rows'] = t['rows_examined_per_scan']
-      res['filtered'] = t['filtered']
-
-      if t['possible_keys'] && t['possible_keys'] != [res['key']]
-        res['possible_keys'] = t['possible_keys']
-      end
-      res['using_index'] = t['using_index'] if t['using_index']
-
-      res.merge!(extra)
-
-      res
-    end
-
-    def self.transform_json(json, res = [], extra = {})
-      rows = []
-
-      if (ordering = json['ordering_operation'])
-        index_walk = (ordering['using_filesort'] == false)
-        return transform_json(json['ordering_operation'], res, { "index_walk" => index_walk } )
-      elsif json['duplicates_removal']
-        return transform_json(json['duplicates_removal'], res, extra)
-      elsif json['grouping_operation']
-        return transform_json(json['grouping_operation'], res, extra)
-      elsif !json['nested_loop'] && !json['table']
-        return [{'Extra' => json['message']}]
-      elsif json['nested_loop']
-        json['nested_loop'].map do |nested|
-          transform_json(nested, res, extra)
-        end
-      elsif json['table']
-        res << transform_table(json['table'], extra)
-      end
-      res
-    end
-
     # [{"id"=>1, "select_type"=>"SIMPLE", "table"=>"interwiki", "partitions"=>nil, "type"=>"const", "possible_keys"=>"PRIMARY", "key"=>"PRIMARY", "key_len"=>"34", "ref"=>"const", "rows"=>1, "filtered"=>100.0, "Extra"=>nil}]
     attr_reader :cost
 
@@ -143,7 +107,8 @@ module Shiba
 
     # TODO: need to parse SQL here I think
     def simple_table_scan?
-      @rows.size == 1 && first['using_index'] && (@sql !~ /order by/i)
+      @rows.size == 1 &&  (@sql !~ /order by/i) &&
+        (first['using_index'] || !(@sql =~ /\s+WHERE\s+/i))
     end
 
     def severity
@@ -216,15 +181,14 @@ module Shiba
       messages << "fuzzed_data" if fuzzed?(first_table)
     end
 
+    # TODO: we don't catch some cases like SELECT * from foo where index_col = 1 limit 1
+    # bcs we really just need to parse the SQL.
     check :check_simple_table_scan
     def check_simple_table_scan
       if simple_table_scan?
         if limit
-          messages << 'limited_tablescan'
+          messages << 'limited_scan'
           @cost = limit
-        else
-          tag_query_type
-          @cost = @stats.estimate_key(first_table, first_key, first['used_key_parts'])
         end
       end
     end
@@ -349,9 +313,10 @@ module Shiba
     end
 
     def humanized_explain
-      h = @explain_json['query_block'].dup
-      %w(select_id cost_info).each { |i| h.delete(i) }
-      h
+      #h = @explain_json['query_block'].dup
+      #%w(select_id cost_info).each { |i| h.delete(i) }
+      #h
+      @explain_json
     end
   end
 end

data/lib/shiba/explain/mysql_explain.rb

@@ -0,0 +1,47 @@
+module Shiba
+  class Explain
+    class MysqlExplain
+      def transform_table(table, extra = {})
+        t = table
+        res = {}
+        res['table'] = t['table_name']
+        res['access_type'] = t['access_type']
+        res['key'] = t['key']
+        res['used_key_parts'] = t['used_key_parts'] if t['used_key_parts']
+        res['rows'] = t['rows_examined_per_scan']
+        res['filtered'] = t['filtered']
+
+        if t['possible_keys'] && t['possible_keys'] != [res['key']]
+          res['possible_keys'] = t['possible_keys']
+        end
+        res['using_index'] = t['using_index'] if t['using_index']
+
+        res.merge!(extra)
+
+        res
+      end
+
+      def transform_json(json, res = [], extra = {})
+        rows = []
+
+        if (ordering = json['ordering_operation'])
+          index_walk = (ordering['using_filesort'] == false)
+          return transform_json(json['ordering_operation'], res, { "index_walk" => index_walk } )
+        elsif json['duplicates_removal']
+          return transform_json(json['duplicates_removal'], res, extra)
+        elsif json['grouping_operation']
+          return transform_json(json['grouping_operation'], res, extra)
+        elsif !json['nested_loop'] && !json['table']
+          return [{'Extra' => json['message']}]
+        elsif json['nested_loop']
+          json['nested_loop'].map do |nested|
+            transform_json(nested, res, extra)
+          end
+        elsif json['table']
+          res << transform_table(json['table'], extra)
+        end
+        res
+      end
+    end
+  end
+end

data/lib/shiba/explain/postgres_explain.rb

@@ -0,0 +1,91 @@
+require 'shiba/explain/postgres_explain_index_conditions'
+
+module Shiba
+  class Explain
+    class PostgresExplain
+      def initialize(json)
+        @json = json
+        @state = {}
+      end
+
+      def with_state(hash)
+        old_state = @state
+        @state = @state.merge(hash)
+        yield
+        @state = old_state
+      end
+
+      def transform_node(node, array)
+        case node['Node Type']
+        when "Limit", "LockRows", "Aggregate", "Unique", "Sort", "Hash", "ProjectSet"
+          recurse_plans(node, array)
+        when "Nested Loop"
+          with_state(join_type: node["Join Type"]) do
+            recurse_plans(node, array)
+          end
+        when "Hash Join"
+          join_fields = extract_join_key_parts(node['Hash Cond'])
+          with_state(join_fields: join_fields, join_type: "Hash") do
+            recurse_plans(node, array)
+          end
+        when "Bitmap Heap Scan"
+          with_state(table: node['Relation Name']) do
+            recurse_plans(node, array)
+          end
+        when "Seq Scan"
+          array << {
+            "table" => node["Relation Name"],
+            "access_type" => "ALL",
+            "key" => nil,
+            "filter" => node["Filter"]
+          }
+        when "Index Scan", "Bitmap Index Scan", "Index Only Scan"
+          table = node["Relation Name"] || @state[:table]
+
+          if node['Index Cond']
+            used_key_parts = extract_used_key_parts(node['Index Cond'])
+          else
+            used_key_parts = []
+          end
+
+          h = {
+            "table" => node["Relation Name"] || @state[:table],
+            "access_type" => "ref",
+            "key" => node["Index Name"],
+            "used_key_parts" => used_key_parts
+          }
+
+          if node['Node Type'] == "Index Only Scan"
+            h['using_index'] = true
+          end
+
+          array << h
+        else
+          raise "unhandled node: #{node}"
+        end
+        array
+      end
+
+      def extract_used_key_parts(cond)
+        conds = PostgresExplainIndexConditions.new(cond)
+        conds.fields
+      end
+
+      def extract_join_key_parts(cond)
+        conds = PostgresExplainIndexConditions.new(cond)
+        conds.join_fields
+      end
+
+      def recurse_plans(node, array)
+        node['Plans'].each do |n|
+          transform_node(n, array)
+        end
+      end
+
+      def transform
+        plan = @json.first['Plan']
+        transform_node(plan, [])
+      end
+    end
+  end
+end

data/lib/shiba/explain/postgres_explain_index_conditions.rb

@@ -0,0 +1,137 @@
+require 'strscan'
+
+module Shiba
+  class Explain
+    class PostgresExplainIndexConditions
+      def initialize(string)
+        @string = string
+        @sc = StringScanner.new(string)
+        @fields = nil
+      end
+
+      attr_reader :sc
+      def parse!
+        return if @fields
+        @fields = {}
+        sc.scan(LPAREN)
+        if sc.peek(1) == "(" && !sc.match?(/\(\w+\)::/)
+
+          while sc.peek(1) == "("
+            sc.getch
+            extract_field(sc)
+            sc.scan(/\s+AND\s+/)
+          end
+        else
+          extract_field(sc)
+        end
+      end
+
+      def fields
+        parse!
+        @fields[nil]
+      end
+
+      def join_fields
+        parse!
+        @fields
+      end
+
+      private
+      LPAREN = /\(/
+      RPAREN = /\)/
+
+      def parse_string(sc)
+        v = ""
+        qchar = sc.getch
+        double_quote = qchar * 2
+        while true
+          if sc.peek(1) == qchar
+            if sc.peek(2) == double_quote
+              sc.scan(/#{double_quote}/)
+            else
+              # end of string
+              sc.getch
+              # optional type hint
+              sc.scan(/::\w+(\[\])?/)
+              return v
+            end
+          end
+          v += sc.getch
+        end
+      end
+
+      def parse_value(sc)
+        peek = sc.peek(1)
+        if peek == "'"
+          parse_string(sc)
+        elsif peek == '"'
+          parse_field(sc)
+        elsif (v = sc.scan(/\d+\.?\d*/))
+          if v.include?('.')
+            v.to_f
+          else
+            v.to_i
+          end
+        elsif sc.scan(/ANY \(/)
+          # parse as string
+          v = parse_value(sc)
+          sc.scan(/\)/)
+          v
+        else
+          parse_field(sc)
+        end
+      end
+
+      def parse_ident(sc)
+        peek = sc.peek(1)
+        if peek == "("
+          sc.getch
+          # typed column like (name)::text = 'ben'
+          ident = sc.scan(/[^\)]+/)
+          sc.scan(/\)::\S+/)
+        elsif peek == '"'
+          ident = parse_string(sc)
+        else
+          ident = sc.scan(/[^ \.\)\[]+/)
+          # field[1] for array fields, not bothering to do brace matching here yet, oy vey
+          sc.scan(/\[.*?\]/)
+        end
+        ident
+      end
+
+      def parse_field(sc)
+        first = nil
+        second = nil
+
+        first = parse_ident(sc)
+        if sc.scan(/\./)
+          second = parse_ident(sc)
+          table = first
+          field = second
+        else
+          table = nil
+          field = first
+        end
+
+        @fields[table] ||= []
+        @fields[table] << field unless @fields[table].include?(field)
+      end
+
+
+      def extract_field(sc)
+        # (type = 1)
+        # ((type)::text = 1)
+        # (((type)::text = ANY ('{User,AnonymousUser}'::text[])) AND ((type)::text = 'User'::text))
+        table = nil
+
+        parse_field(sc)
+        sc.scan(/\s+\S+\s+/) # operator
+        parse_value(sc)
+
+        if sc.scan(RPAREN).nil?
+          raise "bad scan; #{sc.inspect}"
+        end
+      end
+    end
+  end
+end