suggest-db-indices 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,16 @@
|
|
1
|
+
# Learnings from Clojure for make great benefit ruby
|
2
|
+
|
3
|
+
module Enumerable
|
4
|
+
# E.g., {:foo => {:bar => 5}}.get_in(:foo, :bar) #=> 5
|
5
|
+
def get_in keys, default = nil
|
6
|
+
v = self[keys.first]
|
7
|
+
rest = keys.drop 1
|
8
|
+
if rest.any?
|
9
|
+
v.get_in rest, default
|
10
|
+
else
|
11
|
+
v ? v : default
|
12
|
+
end
|
13
|
+
rescue NoMethodError => ex
|
14
|
+
default
|
15
|
+
end
|
16
|
+
end
|
@@ -4,7 +4,7 @@ module SuggestDbIndices
|
|
4
4
|
@indexed_columns_by_table ||= connection.tables.reduce({}) do |h, table_name|
|
5
5
|
# Note: can index on multiple columns, which complicates things. Assuming user has done
|
6
6
|
# this correctly for now...
|
7
|
-
h.merge table_name => connection.indexes(table_name).map {|index| index.columns}.flatten
|
7
|
+
h.merge table_name => connection.indexes(table_name).map { |index| index.columns }.flatten
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
@@ -47,45 +47,90 @@ module SuggestDbIndices
|
|
47
47
|
|
48
48
|
def unindexed_foreign_key_columns_by_table
|
49
49
|
unindexed_columns_by_table.reduce({}) do |h, (table, columns)|
|
50
|
-
h.merge table => columns.select {|col| foreign_key?(col) }
|
50
|
+
h.merge table => columns.select { |col| foreign_key?(col) }
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
-
def
|
55
|
-
@config
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
54
|
+
def config
|
55
|
+
@config ||= default_config
|
56
|
+
end
|
57
|
+
|
58
|
+
def go! opts = {}
|
59
|
+
@config = opts.reduce(default_config) { |h, (k, v)| h.merge k => v }
|
60
|
+
table_col_pair_attributes = hash_of_hashes
|
61
|
+
|
62
|
+
unindexed_foreign_key_columns_by_table.each do |table, cols|
|
63
|
+
cols.each do |col|
|
64
|
+
table_col_pair = [table, col]
|
65
|
+
table_col_pair_attributes[table_col_pair][:foreign_key_column] = true
|
60
66
|
end
|
61
67
|
end
|
62
|
-
|
68
|
+
|
69
|
+
scan_result = scan_log_files
|
70
|
+
columns_found_in_logs_count_by_table = scan_result[:queried_columns_by_table]
|
71
|
+
columns_found_in_logs_count_by_table.each do |table, column_hashes|
|
72
|
+
column_hashes.each do |col, found_count|
|
73
|
+
table_col_pair = [table, col]
|
74
|
+
table_col_pair_attributes[table_col_pair][:found_count] = found_count
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
table_col_pair_validator = @config[:mode] == :conservative \
|
79
|
+
? lambda { |_, attributes| attributes[:foreign_key_column] && attributes[:found_count] }
|
80
|
+
: lambda { |_, attributes| attributes[:foreign_key_column] }
|
81
|
+
|
82
|
+
table_col_pairs_to_index_with_attributes =
|
83
|
+
table_col_pair_attributes.select &table_col_pair_validator
|
84
|
+
|
85
|
+
if table_col_pairs_to_index_with_attributes.any?
|
86
|
+
generate_migration_file! format_index_migration_string table_col_pairs_to_index_with_attributes
|
87
|
+
else
|
88
|
+
puts "No missing indexes found!"
|
89
|
+
end
|
63
90
|
end
|
64
91
|
|
65
|
-
def format_index_migration_string
|
66
|
-
add_index_statements =
|
67
|
-
|
92
|
+
def format_index_migration_string table_col_pairs_to_index_with_attributes
|
93
|
+
add_index_statements = table_col_pairs_to_index_with_attributes.reduce('') do |s, (table_col_pair, attributes)|
|
94
|
+
table, col = table_col_pair
|
95
|
+
s += " add_index :#{table}, :#{col} #"
|
96
|
+
comments = []
|
97
|
+
comments << "foreign key" if attributes[:foreign_key_column]
|
98
|
+
comments << "found in queries #{attributes[:found_count]} times" if attributes[:found_count]
|
99
|
+
s += "#{comments.join(', ')}\n"
|
68
100
|
s
|
69
101
|
end
|
70
102
|
" def change\n#{add_index_statements}\n end\nend"
|
71
103
|
end
|
72
104
|
|
105
|
+
def name_migration_file
|
106
|
+
name = "add_indexes_via_suggest_db_indices"
|
107
|
+
existing_migration_files = Dir.glob File.join Rails.root, 'db', 'migrate/*.rb'
|
108
|
+
|
109
|
+
if existing_migration_files.any? { |f| f.end_with?("#{name}.rb") }
|
110
|
+
i = 1
|
111
|
+
i += 1 while existing_migration_files.any? { |f| f.end_with?("#{name}_#{i}.rb") }
|
112
|
+
name += "_#{i}"
|
113
|
+
end
|
114
|
+
name
|
115
|
+
end
|
116
|
+
|
73
117
|
def generate_migration_file! migration_contents
|
74
|
-
_
|
75
|
-
|
76
|
-
|
118
|
+
_, migration_file_path = Rails::Generators.invoke("active_record:migration",
|
119
|
+
[name_migration_file,
|
120
|
+
'BoiledGoose:Animal']) # Bogus param, doesn't matter since contents will be replaced
|
77
121
|
file_contents = File.read migration_file_path
|
78
122
|
search_string = "ActiveRecord::Migration"
|
79
123
|
stop_index = (file_contents.index(search_string)) + search_string.length
|
80
124
|
new_file_contents = file_contents[0..stop_index] + migration_contents
|
81
|
-
File.open(migration_file_path, 'w') {|f| f.write(new_file_contents) }
|
125
|
+
File.open(migration_file_path, 'w') { |f| f.write(new_file_contents) }
|
126
|
+
puts "Migration result: \n #{new_file_contents}"
|
82
127
|
migration_file_path
|
83
128
|
end
|
84
129
|
|
85
|
-
def
|
130
|
+
def default_config
|
86
131
|
{:num_lines_to_scan => 10000,
|
87
|
-
|
88
|
-
|
132
|
+
:examine_logs => false,
|
133
|
+
:log_dir => File.join(Rails.root, 'log')}
|
89
134
|
end
|
90
135
|
|
91
136
|
def prepare_log_file! log_dir
|
@@ -95,48 +140,52 @@ module SuggestDbIndices
|
|
95
140
|
puts "Found log files: #{log_file_names.inspect}"
|
96
141
|
|
97
142
|
puts "Tailing each file!"
|
98
|
-
log_file_names.each {|f| sh_dbg "tail -n #{
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
143
|
+
log_file_names.each { |f| sh_dbg "tail -n #{config[:num_lines_to_scan]} #{f} >> #{tmpfile.path}" }
|
144
|
+
tmpfile
|
145
|
+
end
|
146
|
+
|
147
|
+
def table_quote_char
|
148
|
+
@table_quote_char ||= connection.quote_table_name("boiled_goose")[0]
|
104
149
|
end
|
105
150
|
|
106
|
-
def
|
151
|
+
def column_quote_char
|
152
|
+
@column_quote_char ||= connection.quote_column_name("sauerkraut")[0]
|
153
|
+
end
|
154
|
+
|
155
|
+
# Scans log files for queried columns
|
156
|
+
def scan_log_files log_dir = config()[:log_dir]
|
107
157
|
stripped_log_file = prepare_log_file! log_dir
|
108
158
|
|
109
|
-
queried_columns_by_table =
|
110
|
-
# For debugging: Record from
|
159
|
+
queried_columns_by_table = hash_of_hashes
|
160
|
+
# For debugging: Record from what table and columns we derived from each SQL statement
|
111
161
|
inferred_table_columns_by_raw_where_clause = hash_of_sets
|
162
|
+
non_matches = Set.new
|
112
163
|
|
113
164
|
while line = stripped_log_file.gets
|
114
165
|
line = remove_limit_clause(line.strip)
|
115
|
-
if matches = /SELECT.+FROM\s\W?(\w+)\W?\sWHERE(.+)/
|
116
|
-
|
117
|
-
|
118
|
-
raw_where_clause
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
s.gsub('`','')
|
123
|
-
end.reduce([]) do |memo, identifier| #TODO: Stop reducing to array, reduce to counter
|
124
|
-
if identifier.include?('.')
|
166
|
+
if matches = /SELECT.+WHERE(.+)/i.match(line) #Old: /.+SELECT.+FROM\s\W?(\w+)\W?\sWHERE(.+)/
|
167
|
+
raw_where_clause = matches[1]
|
168
|
+
# puts "Where: #{raw_where_clause}"
|
169
|
+
raw_where_clause.split.each do |identifier|
|
170
|
+
next if non_matches.include? identifier
|
171
|
+
# Go through the where clause to find columns that were queried
|
172
|
+
if identifier.include?('.') # e.g., "post"."user_id"
|
125
173
|
current_table, column_candidate = identifier.split('.')
|
126
|
-
|
127
|
-
|
174
|
+
current_table.gsub! table_quote_char, ''
|
175
|
+
column_candidate.gsub! column_quote_char, ''
|
176
|
+
if non_pk_columns_by_table[current_table] && non_pk_columns_by_table[current_table].include?(column_candidate)
|
177
|
+
# We only care about the identifiers that match up to a table and column.
|
178
|
+
# This is a ghetto way to to avoid having to parse SQL (extremely difficult)
|
179
|
+
if queried_columns_by_table.get_in([current_table, column_candidate])
|
180
|
+
queried_columns_by_table[current_table][column_candidate] += 1
|
181
|
+
else
|
182
|
+
queried_columns_by_table[current_table] = {column_candidate => 1}
|
183
|
+
end
|
184
|
+
inferred_table_columns_by_raw_where_clause[raw_where_clause] << [current_table, column_candidate]
|
185
|
+
else
|
186
|
+
non_matches << identifier
|
187
|
+
end
|
128
188
|
end
|
129
|
-
|
130
|
-
if non_pk_columns_by_table[current_table].include? column_candidate
|
131
|
-
# We only care about the identifiers that match up to a table and column.
|
132
|
-
# This is a ghetto way to to avoid having to parse SQL (extremely difficult)
|
133
|
-
memo << [current_table, column_candidate]
|
134
|
-
else
|
135
|
-
memo
|
136
|
-
end
|
137
|
-
end.each do |(table, column)|
|
138
|
-
queried_columns_by_table[table] << column
|
139
|
-
inferred_table_columns_by_raw_where_clause[raw_where_clause] << [table, column]
|
140
189
|
end
|
141
190
|
end
|
142
191
|
end
|
@@ -145,11 +194,15 @@ module SuggestDbIndices
|
|
145
194
|
end
|
146
195
|
|
147
196
|
def hash_of_arrays
|
148
|
-
Hash.new {|h, k| h[k] = [] }
|
197
|
+
Hash.new { |h, k| h[k] = [] }
|
198
|
+
end
|
199
|
+
|
200
|
+
def hash_of_hashes
|
201
|
+
Hash.new { |h, k| h[k] = Hash.new }
|
149
202
|
end
|
150
203
|
|
151
204
|
def hash_of_sets
|
152
|
-
Hash.new {|h, k| h[k] = Set.new }
|
205
|
+
Hash.new { |h, k| h[k] = Set.new }
|
153
206
|
end
|
154
207
|
|
155
208
|
def remove_limit_clause s
|
@@ -160,12 +213,6 @@ module SuggestDbIndices
|
|
160
213
|
end
|
161
214
|
end
|
162
215
|
|
163
|
-
def strip_color_codes! file_name, output_path
|
164
|
-
# From: http://serverfault.com/a/154200
|
165
|
-
sh_dbg 'sed "s/${esc}[^m]*m//g" ' + "#{file_name} >> #{output_path}"
|
166
|
-
raise "There was a problem stripping colors" unless $?.success?
|
167
|
-
end
|
168
|
-
|
169
216
|
def sh_dbg cmd
|
170
217
|
puts "Shelling: #{cmd}"
|
171
218
|
`#{cmd}`
|
data/lib/suggest_db_indices.rb
CHANGED
@@ -2,4 +2,5 @@ require 'tempfile'
|
|
2
2
|
require 'rails/generators'
|
3
3
|
|
4
4
|
require File.join File.dirname(__FILE__), '/suggest_db_indices/version'
|
5
|
+
require File.join File.dirname(__FILE__), '/suggest_db_indices/clojurian_imperialism'
|
5
6
|
require File.join File.dirname(__FILE__), '/suggest_db_indices/core'
|
data/suggest_db_indices.gemspec
CHANGED
@@ -2,19 +2,21 @@
|
|
2
2
|
require File.join File.dirname(__FILE__), 'lib/suggest_db_indices/version'
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
|
-
gem.authors
|
6
|
-
gem.email
|
7
|
-
gem.description
|
8
|
-
gem.summary
|
9
|
-
gem.homepage
|
10
|
-
gem.license
|
5
|
+
gem.authors = ["Isak Sky"]
|
6
|
+
gem.email = ["isak.sky@gmail.com"]
|
7
|
+
gem.description = %q{A gem for your rails project that suggests indices for you to add in your database. Currently it suggests adding indices to unindexed foreign key columns.}
|
8
|
+
gem.summary = %q{A gem for your rails project that suggests indices for you to add in your database.}
|
9
|
+
gem.homepage = "https://github.com/isaksky/suggest-db-indices"
|
10
|
+
gem.license = "MIT"
|
11
11
|
|
12
|
-
gem.files
|
13
|
-
gem.executables
|
14
|
-
gem.test_files
|
15
|
-
gem.name
|
12
|
+
gem.files = `git ls-files`.split($\)
|
13
|
+
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
14
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
15
|
+
gem.name = "suggest-db-indices"
|
16
16
|
gem.require_paths = ["lib"]
|
17
|
-
gem.version
|
17
|
+
gem.version = SuggestDbIndices::VERSION
|
18
18
|
|
19
19
|
gem.add_dependency('rails', '>= 3.0.0')
|
20
|
+
gem.add_development_dependency('awesome_print')
|
21
|
+
gem.add_development_dependency('pry')
|
20
22
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: suggest-db-indices
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-07-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
16
|
-
requirement: &
|
16
|
+
requirement: &2161081920 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,7 +21,29 @@ dependencies:
|
|
21
21
|
version: 3.0.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *2161081920
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: awesome_print
|
27
|
+
requirement: &2161081300 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :development
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *2161081300
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: pry
|
38
|
+
requirement: &2161080600 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :development
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *2161080600
|
25
47
|
description: A gem for your rails project that suggests indices for you to add in
|
26
48
|
your database. Currently it suggests adding indices to unindexed foreign key columns.
|
27
49
|
email:
|
@@ -36,6 +58,7 @@ files:
|
|
36
58
|
- README.md
|
37
59
|
- Rakefile
|
38
60
|
- lib/suggest_db_indices.rb
|
61
|
+
- lib/suggest_db_indices/clojurian_imperialism.rb
|
39
62
|
- lib/suggest_db_indices/core.rb
|
40
63
|
- lib/suggest_db_indices/version.rb
|
41
64
|
- suggest_db_indices.gemspec
|