shiba 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,7 +13,7 @@ module Shiba
13
13
  @tables.any?
14
14
  end
15
15
 
16
- Table = Struct.new(:name, :count, :indexes, :average_row_size) do
16
+ Table = Struct.new(:name, :count, :indexes) do
17
17
  def encode_with(coder)
18
18
  coder.map = self.to_h.stringify_keys
19
19
  coder.map.delete('name')
@@ -23,6 +23,7 @@ module Shiba
23
23
  self.count = indexes.map { |i, parts| parts.columns.map { |v| v.raw_cardinality } }.flatten.max
24
24
  end
25
25
 
26
+ coder.map['column_sizes'] = column_sizes
26
27
  coder.tag = nil
27
28
  end
28
29
 
@@ -39,6 +40,10 @@ module Shiba
39
40
  self.count = cardinality
40
41
  end
41
42
  end
43
+
44
+ def column_sizes
45
+ @column_sizes ||= {}
46
+ end
42
47
  end
43
48
 
44
49
  Index = Struct.new(:table, :name, :columns, :unique) do
@@ -152,6 +157,20 @@ module Shiba
152
157
  table.add_index_column(index_name, column_name, nil, cardinality, is_unique)
153
158
  end
154
159
 
160
+ def get_column_size(table_name, column)
161
+ table = @tables[table_name]
162
+ return nil unless table
163
+
164
+ table.column_sizes[column]
165
+ end
166
+
167
+ def set_column_size(table_name, column, size)
168
+ table = @tables[table_name]
169
+ raise "couldn't find table: #{table_name}" unless table
170
+
171
+ table.column_sizes[column] = size
172
+ end
173
+
155
174
  def estimate_key(table_name, key, parts)
156
175
  index = fetch_index(table_name, key)
157
176
 
@@ -71,4 +71,4 @@ index_walk:
71
71
  level: success
72
72
  retsize:
73
73
  title: Results
74
- summary: The database returns {{ result_size }} row(s) to the client.
74
+ summary: The database returns {{ formatted_result }} to the client.
@@ -0,0 +1,64 @@
1
+ require 'shiba/parsers/shiba_string_scanner'
2
+
3
+ module Shiba
4
+ module Parsers
5
+ class MysqlSelectFields
6
+ def initialize(sql)
7
+ @sql = sql
8
+ @sc = ShibaStringScanner.new(@sql)
9
+ end
10
+ attr_reader :sc
11
+
12
+ BACKTICK = "`"
13
+
14
+ def tick_match
15
+ sc.match_quoted_double_escape(BACKTICK)
16
+ end
17
+
18
+ def parse_fields
19
+ tables = {}
20
+
21
+ sc.scan(%r{/\*.*?\*/ select })
22
+
23
+ while !sc.scan(/ from/i)
24
+ sc.scan(/distinct /)
25
+
26
+ if sc.scan(/\w+\(/)
27
+ parens = 1
28
+ while parens > 0
29
+ case sc.getch
30
+ when '('
31
+ parens += 1
32
+ when ')'
33
+ parens -= 1
34
+ end
35
+ end
36
+ sc.scan(/ AS /)
37
+ tick_match
38
+ # parse function
39
+ elsif sc.scan(/`(.*?)`\.`(.*?)`\.`(.*?)` AS `(.*?)`/)
40
+ db = sc[1]
41
+ table = sc[2]
42
+ col = sc[3]
43
+
44
+ tables[table] ||= []
45
+ tables[table] << col
46
+
47
+ elsif sc.scan(/`(.*?)`\.`(.*?)` AS `(.*?)`/)
48
+ table = sc[1]
49
+ col = sc[2]
50
+
51
+ tables[table] ||= []
52
+ tables[table] << col
53
+ elsif sc.scan(/(\d+|NULL|'.*?') AS `(.*?)`/m)
54
+ else
55
+ raise "unknown stuff: in #{@sql}: #{@sc.rest}"
56
+ end
57
+
58
+ sc.scan(/,/)
59
+ end
60
+ tables
61
+ end
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,27 @@
1
+ require 'strscan'
2
+
3
+ module Shiba
4
+ module Parsers
5
+ class ShibaStringScanner < StringScanner
6
+ def match_quoted_double_escape(quote)
7
+ getch
8
+
9
+ str = ""
10
+ while ch = getch
11
+ if ch == quote
12
+ if peek(1) == quote
13
+ str += ch
14
+ str += getch
15
+ else
16
+ return str
17
+ end
18
+ else
19
+ str += ch
20
+ end
21
+ end
22
+ str
23
+ end
24
+ end
25
+ end
26
+ end
27
+
@@ -0,0 +1,196 @@
1
+ module Shiba
2
+ module Review
3
+ # Builds options for interacting with the reviewer via the command line.
4
+ # Automatically infers options from environment variables on CI.
5
+ #
6
+ # Example:
7
+ # cli = CLI.new
8
+ # cli.valid?
9
+ # => true
10
+ # cli.options
11
+ # => { "file" => 'path/to/explain_log.json' }
12
+ #
13
+ # or
14
+ #
15
+ # cli.valid?
16
+ # => false
17
+ # cli.failure
18
+ # => "An error message with command line help."
19
+ class CLI
20
+
21
+ attr_reader :errors
22
+
23
+ def initialize
24
+ @user_options = {}
25
+ @errors = []
26
+ parser.parse!
27
+ @options = default_options.merge(@user_options)
28
+ end
29
+
30
+ def options
31
+ @options
32
+ end
33
+
34
+ def valid?
35
+ return false if @errors.any?
36
+
37
+ validate_log_path
38
+ #validate_git_repo if branch || options["submit"]
39
+ description = "Provide an explain log, or run 'shiba explain' to generate one."
40
+
41
+ require_option("file", description: description)
42
+
43
+ if options["submit"]
44
+ require_option("branch") if options["diff"].nil?
45
+ require_option("token")
46
+ require_option("pull_request")
47
+ end
48
+
49
+ @errors.empty?
50
+ end
51
+
52
+ def failure
53
+ return nil if @errors.empty?
54
+
55
+ message, help = @errors.first
56
+ message += "\n"
57
+ if help
58
+ message += "\n#{parser}"
59
+ end
60
+
61
+ message
62
+ end
63
+
64
+ def report_options(*keys)
65
+ keys.each do |key|
66
+ report("#{key}: #{options[key]}")
67
+ end
68
+ end
69
+
70
+ protected
71
+
72
+ def parser
73
+ @parser ||= OptionParser.new do |opts|
74
+ opts.banner = "Review changes for query problems. Optionally submit the comments to a Github pull request."
75
+
76
+ opts.separator "Required:"
77
+
78
+ opts.on("-f","--file FILE", "The explain output log to compare with. Automatically configured when $CI environment variable is set") do |f|
79
+ @user_options["file"] = f
80
+ end
81
+
82
+ opts.separator ""
83
+ opts.separator "Git diff options:"
84
+
85
+ opts.on("-b", "--branch GIT_BRANCH", "Compare to changes between origin/HEAD and BRANCH. Attempts to read from CI environment when not set.") do |b|
86
+ @user_options["branch"] = b
87
+ end
88
+
89
+ opts.on("--staged", "Only check files that are staged for commit") do
90
+ @user_options["staged"] = true
91
+ end
92
+
93
+ opts.on("--unstaged", "Only check files that are not staged for commit") do
94
+ @user_options["unstaged"] = true
95
+ end
96
+
97
+ opts.separator ""
98
+ opts.separator "Github options:"
99
+
100
+ opts.on("--submit", "Submit comments to Github") do
101
+ @user_options["submit"] = true
102
+ end
103
+
104
+ opts.on("-p", "--pull-request PR_ID", "The ID of the pull request to comment on. Attempts to read from CI environment when not set.") do |p|
105
+ @user_options["pull_request"] = p
106
+ end
107
+
108
+ opts.on("-t", "--token TOKEN", "The Github API token to use for commenting. Defaults to $GITHUB_TOKEN.") do |t|
109
+ @user_options["token"] = t
110
+ end
111
+
112
+ opts.separator ""
113
+ opts.separator "Common options:"
114
+
115
+ opts.on("--verbose", "Verbose/debug mode") do
116
+ @user_options["verbose"] = true
117
+ end
118
+
119
+ opts.on_tail("-h", "--help", "Show this message") do
120
+ puts opts
121
+ exit
122
+ end
123
+
124
+ opts.on_tail("--version", "Show version") do
125
+ require 'shiba/version'
126
+ puts Shiba::VERSION
127
+ exit
128
+ end
129
+ end
130
+ end
131
+
132
+ def default_options
133
+ defaults = {}
134
+
135
+ if ENV['DIFF']
136
+ defaults['diff'] = ENV['DIFF']
137
+ end
138
+
139
+ if Shiba::Configure.ci?
140
+ report("Finding default options from CI environment.")
141
+
142
+ defaults["file"] = ci_explain_log_path
143
+ defaults["pull_request"] = ci_pull_request
144
+ defaults["branch"] = ci_branch if !defaults['diff']
145
+ end
146
+
147
+ defaults["token"] = ENV['GITHUB_TOKEN'] if ENV['GITHUB_TOKEN']
148
+
149
+ defaults
150
+ end
151
+
152
+ def validate_log_path
153
+ return if !options["file"]
154
+ if !File.exist?(options["file"])
155
+ error("File not found: '#{options["file"]}'")
156
+ end
157
+ end
158
+
159
+ def ci_explain_log_path
160
+ File.join(Shiba.path, 'ci.json')
161
+ end
162
+
163
+ def ci_branch
164
+ ENV['TRAVIS_PULL_REQUEST_SHA'] || ENV['CIRCLE_SHA1']
165
+ end
166
+
167
+ def ci_pull_request
168
+ ENV['TRAVIS_PULL_REQUEST'] || circle_pr_number
169
+ end
170
+
171
+ def circle_pr_number
172
+ return if ENV["CIRCLE_PULL_REQUEST"].nil?
173
+ number = URI.parse(ENV["CIRCLE_PULL_REQUEST"]).path.split("/").last
174
+ return if number !~ /\A[0-9]+\Z/
175
+
176
+ number
177
+ end
178
+
179
+ def require_option(name, description: nil)
180
+ return true if options.key?(name)
181
+ msg = "Required: '#{name}'"
182
+ msg << ". #{description}" if description
183
+ error(msg, help: true)
184
+ end
185
+
186
+ def report(message)
187
+ $stderr.puts message if @user_options["verbose"]
188
+ end
189
+
190
+ def error(message, help: false)
191
+ @errors << [ message, help ]
192
+ end
193
+
194
+ end
195
+ end
196
+ end
@@ -43,15 +43,32 @@ module Shiba
43
43
  "fuzz_table_sizes" => fuzzed_sizes(message),
44
44
  "table" => message["table"],
45
45
  "table_size" => message["table_size"],
46
- "result_size" => message["result_size"],
47
46
  "index" => message["index"],
48
47
  "join_to" => message["join_to"],
49
48
  "key_parts" => (message["index_used"] || []).join(','),
50
49
  "size" => message["size"],
51
- "formatted_cost" => formatted_cost(message)
50
+ "formatted_cost" => formatted_cost(message),
51
+ "formatted_result" => formatted_result(message)
52
52
  }
53
53
  end
54
54
 
55
+ def formatted_result(explain)
56
+ return nil unless explain['result_bytes'] && explain['result_size']
57
+
58
+ bytes = explain['result_bytes']
59
+ return "%d rows" % explain['result_size'] if bytes == 0
60
+
61
+ if bytes < 1000
62
+ result = "%d bytes" % bytes
63
+ elsif bytes < 1000000
64
+ result = "%dkb" % (bytes / 1000)
65
+ else
66
+ result = "%.1fmb" % (bytes / 1000000.0 )
67
+ end
68
+
69
+ "%s (%d rows)" % [result, explain['result_size']]
70
+ end
71
+
55
72
  def formatted_cost(explain)
56
73
  return nil unless explain["rows_read"] && explain["table_size"]
57
74
  percentage = (explain["rows_read"] / explain["table_size"]) * 100.0;
@@ -0,0 +1,227 @@
1
+ module Shiba
2
+ module Review
3
+ module Diff
4
+
5
+ class Parser
6
+ # +++ b/config/environments/test.rb
7
+ FILE_PATTERN = /\A\+\+\+ b\/(.*?)\Z/
8
+
9
+ # @@ -177,0 +178 @@ ...
10
+ # @@ -177,0 +178,5 @@ ...
11
+ # @@ -21 +24 @@ ...
12
+ LINE_PATTERN = /\A@@ \-\d+,?\d+? \+(\d+),?(\d+)? @@/
13
+
14
+ # via https://developer.github.com/v3/pulls/comments/#create-a-comment
15
+ # The position value equals the number of lines down from the first "@@" hunk header
16
+ # in the file you want to add a comment.
17
+
18
+ attr_reader :status
19
+
20
+ def initialize(file)
21
+ # Fixme. seems like enumerables should work in general.
22
+ if !file.respond_to?(:pos)
23
+ raise StandardError.new("Diff file does not appear to be a seekable IO object.")
24
+ end
25
+ @diff = file
26
+ @status = :new
27
+ end
28
+
29
+ # Returns the file and line numbers that contain inserts. Deletions are ignored.
30
+ # For simplicity, the default output of git diff is not supported.
31
+ # The expected format is from 'git diff unified=0'
32
+ #
33
+ # Example:
34
+ # diff = `git diff --unified=0`
35
+ # Diff.new(StringIO.new(diff))
36
+ # => [ [ "hello.rb", 1..3 ]
37
+ # => [ "hello.rb", 7..7 ]
38
+ # => [ "test.rb", 23..23 ]
39
+ # => ]
40
+ def updated_lines
41
+ io = @diff.each_line
42
+ path = nil
43
+
44
+ found = []
45
+
46
+ while true
47
+ line = io.next
48
+ if line =~ FILE_PATTERN
49
+ path = $1
50
+ end
51
+
52
+ if hunk_header?(line)
53
+ line_numbers = line_numbers_for_destination(line)
54
+ found << [ path, line_numbers ]
55
+ end
56
+ end
57
+ rescue StopIteration
58
+ return found
59
+ end
60
+
61
+ # Returns the position in the diff, after the relevant file header,
62
+ # that contains the specified file/lineno modification.
63
+ # Only supports finding the position in the destination / newest version of the file.
64
+ #
65
+ # Example:
66
+ # diff = Diff.new(`git diff`)
67
+ # diff.find_position("test.rb", 3)
68
+ # => 5
69
+ def find_position(path, line_number)
70
+ io = @diff.each_line # maybe redundant?
71
+
72
+ file_header = "+++ b/#{path}\n" # fixme
73
+ if !io.find_index(file_header)
74
+ @status = :file_not_found
75
+ return
76
+ end
77
+
78
+ line = io.peek
79
+ if !hunk_header?(line)
80
+ raise StandardError.new("Expected hunk header to be after file header, but got '#{line}'")
81
+ end
82
+
83
+ pos = 0
84
+
85
+ while true
86
+ line = io.next
87
+ pos += 1
88
+
89
+ if file_header?(line)
90
+ @status = :line_not_found
91
+ return
92
+ end
93
+
94
+ if !hunk_header?(line)
95
+ next
96
+ end
97
+
98
+ line_numbers = line_numbers_for_destination(line)
99
+
100
+ if destination_position = line_numbers.find_index(line_number)
101
+ @status = :found_position
102
+ return pos + find_hunk_index(io, destination_position)
103
+ end
104
+ end
105
+ rescue StopIteration
106
+ @status = :line_not_found
107
+ end
108
+
109
+ protected
110
+
111
+ def find_hunk_index(hunk, pos)
112
+ line, idx = hunk.with_index.select { |l,idx| !l.start_with?('-') }.take(pos+1).last
113
+ idx
114
+ end
115
+
116
+ def file_header?(line)
117
+ line =~ FILE_PATTERN
118
+ end
119
+
120
+ def hunk_header?(line)
121
+ LINE_PATTERN =~ line
122
+ end
123
+
124
+ def line_numbers_for_destination(diff_line)
125
+ diff_line =~ LINE_PATTERN
126
+ line = $1.to_i
127
+ line_count = ($2 && $2.to_i) || 0
128
+ line..line+line_count
129
+ end
130
+ end
131
+
132
+ class GitDiff
133
+
134
+ # Valid options are: "staged", "unstaged", "branch", and "verbose"
135
+ def initialize(options)
136
+ @options = options
137
+ end
138
+
139
+ def paths
140
+ if options['verbose']
141
+ puts cmd
142
+ end
143
+
144
+ run = "git diff#{cmd} --name-only --diff-filter=d"
145
+
146
+ if options[:verbose]
147
+ $stderr.puts run
148
+ end
149
+ result = `#{run}`
150
+ if $?.exitstatus != 0
151
+ $stderr.puts result
152
+ raise Shiba::Error.new "Failed to read changes"
153
+ end
154
+
155
+ result.split("\n")
156
+ end
157
+
158
+ # Differ expects context: 0, ignore_deletions: true
159
+ def file(context: nil, ignore_deletions: false)
160
+ differ = "git diff#{cmd}"
161
+ differ << " --unified=#{context}" if context
162
+ differ << " --diff-filter=d" if ignore_deletions
163
+ run(differ)
164
+ end
165
+
166
+ protected
167
+
168
+ attr_reader :options
169
+
170
+ def run(command)
171
+ if options[:verbose]
172
+ $stderr.puts command
173
+ end
174
+
175
+ _, out,_,_ = Open3.popen3(command)
176
+ out
177
+ end
178
+
179
+ def cmd
180
+ cmd = case
181
+ when options["staged"]
182
+ " --staged"
183
+ when options["unstaged"]
184
+ ""
185
+ else
186
+ commit = " origin/HEAD"
187
+ commit << "...#{options["branch"]}" if options["branch"]
188
+ commit
189
+ end
190
+ end
191
+ end
192
+
193
+ class FileDiff
194
+ # +++ b/test/app/app.rb
195
+ FILE_NAME_PATTERN = /^\+\+\+ b\/(.*?)$/
196
+
197
+ def initialize(path)
198
+ @path = path
199
+ end
200
+
201
+ # Extracts path names from the diff file
202
+ #
203
+ # Example:
204
+ # index ade9b24..661d522 100644
205
+ # --- a/test/app/app.rb
206
+ # +++ b/test/app/app.rb
207
+ # @@ -24,4 +24,4 @@ ActiveRecord::Base...
208
+ # org = Organization.create!(name: 'test')
209
+ #
210
+ # diff.paths
211
+ # => [ test/app/app.rb ]
212
+ def paths
213
+ f = File.open(@path)
214
+ f.grep(FILE_NAME_PATTERN) { $1 }
215
+ end
216
+
217
+ def file(context: nil, ignore_deletions: nil)
218
+ warn "Context not supported for file diffs" if context
219
+ warn "Ignore deletions not supported for file diffs" if ignore_deletions
220
+ File.open(@path)
221
+ end
222
+
223
+ end
224
+
225
+ end
226
+ end
227
+ end