suggest-db-indices 0.0.2 → 0.0.3

data/lib/suggest_db_indices/clojurian_imperialism.rb

@@ -0,0 +1,16 @@
+# Learnings from Clojure for make great benefit ruby
+
+module Enumerable
+  # E.g., {:foo => {:bar => 5}}.get_in(:foo, :bar) #=> 5
+  def get_in keys, default = nil
+    v = self[keys.first]
+    rest = keys.drop 1
+    if rest.any?
+      v.get_in rest, default
+    else
+      v ? v : default
+    end
+  rescue NoMethodError => ex
+    default
+  end
+end

data/lib/suggest_db_indices/core.rb

@@ -4,7 +4,7 @@ module SuggestDbIndices
       @indexed_columns_by_table ||= connection.tables.reduce({}) do |h, table_name|
         # Note: can index on multiple columns, which complicates things.  Assuming user has done
         # this correctly for now...
-        h.merge table_name => connection.indexes(table_name).map {|index| index.columns}.flatten
+        h.merge table_name => connection.indexes(table_name).map { |index| index.columns }.flatten
       end
     end
 
@@ -47,45 +47,90 @@ module SuggestDbIndices
 
     def unindexed_foreign_key_columns_by_table
       unindexed_columns_by_table.reduce({}) do |h, (table, columns)|
-        h.merge table => columns.select {|col| foreign_key?(col) }
+        h.merge table => columns.select { |col| foreign_key?(col) }
       end
     end
 
-    def go! config = {}
-      @config = default_options.reduce(config) do |h, (k,v)|
-        if h[k]
-          h
-        else
-          h.merge! k => v
+    def config
+      @config ||= default_config
+    end
+
+    def go! opts = {}
+      @config = opts.reduce(default_config) { |h, (k, v)| h.merge k => v }
+      table_col_pair_attributes = hash_of_hashes
+
+      unindexed_foreign_key_columns_by_table.each do |table, cols|
+        cols.each do |col|
+          table_col_pair = [table, col]
+          table_col_pair_attributes[table_col_pair][:foreign_key_column] = true
         end
       end
-      generate_migration_file! format_index_migration_string unindexed_foreign_key_columns_by_table
+
+      scan_result = scan_log_files
+      columns_found_in_logs_count_by_table = scan_result[:queried_columns_by_table]
+      columns_found_in_logs_count_by_table.each do |table, column_hashes|
+        column_hashes.each do |col, found_count|
+          table_col_pair = [table, col]
+          table_col_pair_attributes[table_col_pair][:found_count] = found_count
+        end
+      end
+
+      table_col_pair_validator = @config[:mode] == :conservative \
+      ? lambda { |_, attributes| attributes[:foreign_key_column] && attributes[:found_count] }
+      : lambda { |_, attributes| attributes[:foreign_key_column] }
+
+      table_col_pairs_to_index_with_attributes =
+          table_col_pair_attributes.select &table_col_pair_validator
+
+      if table_col_pairs_to_index_with_attributes.any?
+        generate_migration_file! format_index_migration_string table_col_pairs_to_index_with_attributes
+      else
+        puts "No missing indexes found!"
+      end
     end
 
-    def format_index_migration_string columns_by_table
-      add_index_statements = columns_by_table.reduce('') do |s, (table, columns)|
-        columns.each {|col| s += "    add_index :#{table}, :#{col}\n" }
+    def format_index_migration_string table_col_pairs_to_index_with_attributes
+      add_index_statements = table_col_pairs_to_index_with_attributes.reduce('') do |s, (table_col_pair, attributes)|
+        table, col = table_col_pair
+        s += "    add_index :#{table}, :#{col} #"
+        comments = []
+        comments << "foreign key" if attributes[:foreign_key_column]
+        comments << "found in queries #{attributes[:found_count]} times" if attributes[:found_count]
+        s += "#{comments.join(', ')}\n"
         s
       end
       "  def change\n#{add_index_statements}\n  end\nend"
     end
 
+    def name_migration_file
+      name = "add_indexes_via_suggest_db_indices"
+      existing_migration_files = Dir.glob File.join Rails.root, 'db', 'migrate/*.rb'
+
+      if existing_migration_files.any? { |f| f.end_with?("#{name}.rb") }
+        i = 1
+        i += 1 while existing_migration_files.any? { |f| f.end_with?("#{name}_#{i}.rb") }
+        name += "_#{i}"
+      end
+      name
+    end
+
     def generate_migration_file! migration_contents
-      _ , migration_file_path  = Rails::Generators.invoke("active_record:migration",
-                                                          ["add_indexes_via_suggest_db_indices_#{rand(36**8).to_s(36)}",
-                                                           'BoiledGoose:Animal'])
+      _, migration_file_path = Rails::Generators.invoke("active_record:migration",
+                                                        [name_migration_file,
+                                                         'BoiledGoose:Animal']) # Bogus param, doesn't matter since contents will be replaced
       file_contents = File.read migration_file_path
       search_string = "ActiveRecord::Migration"
       stop_index = (file_contents.index(search_string)) + search_string.length
       new_file_contents = file_contents[0..stop_index] + migration_contents
-      File.open(migration_file_path, 'w') {|f| f.write(new_file_contents) }
+      File.open(migration_file_path, 'w') { |f| f.write(new_file_contents) }
+      puts "Migration result: \n #{new_file_contents}"
       migration_file_path
     end
 
-    def default_options
+    def default_config
       {:num_lines_to_scan => 10000,
-        :examine_logs => false,
-        :log_dir => ""}
+       :examine_logs => false,
+       :log_dir => File.join(Rails.root, 'log')}
     end
 
     def prepare_log_file! log_dir
@@ -95,48 +140,52 @@ module SuggestDbIndices
       puts "Found log files: #{log_file_names.inspect}"
 
       puts "Tailing each file!"
-      log_file_names.each {|f| sh_dbg "tail -n #{NUM_LINES_TO_READ} #{f} >> #{tmpfile.path}" }
-      puts "Stripping color codes!"
-      stripped_log_file = Tempfile.new('stripped')
-      # Because text search is too tricky with colors
-      strip_color_codes! tmpfile.path, stripped_log_file.path
-      stripped_log_file
+      log_file_names.each { |f| sh_dbg "tail -n #{config[:num_lines_to_scan]} #{f} >> #{tmpfile.path}" }
+      tmpfile
+    end
+
+    def table_quote_char
+      @table_quote_char ||= connection.quote_table_name("boiled_goose")[0]
     end
 
-    def scan_log_files_for_queried_columns log_dir, non_pk_columns_by_table = non_pk_columns_by_table
+    def column_quote_char
+      @column_quote_char ||= connection.quote_column_name("sauerkraut")[0]
+    end
+
+    # Scans log files for queried columns
+    def scan_log_files log_dir = config()[:log_dir]
       stripped_log_file = prepare_log_file! log_dir
 
-      queried_columns_by_table = hash_of_arrays
-      # For debugging: Record from which SQL statement we got each column
+      queried_columns_by_table = hash_of_hashes
+      # For debugging: Record from what table and columns we derived from each SQL statement
       inferred_table_columns_by_raw_where_clause = hash_of_sets
+      non_matches = Set.new
 
       while line = stripped_log_file.gets
         line = remove_limit_clause(line.strip)
-        if matches = /SELECT.+FROM\s\W?(\w+)\W?\sWHERE(.+)/i.match(line)
-          table = matches[1]
-
-          raw_where_clause = matches[2]
-#          puts "Where: #{raw_where_clause}"
-
-          raw_where_clause.split.map do |s|
-            s.gsub('`','')
-          end.reduce([]) do |memo, identifier| #TODO: Stop reducing to array, reduce to counter
-            if identifier.include?('.')
+        if matches = /SELECT.+WHERE(.+)/i.match(line) #Old: /.+SELECT.+FROM\s\W?(\w+)\W?\sWHERE(.+)/
+          raw_where_clause = matches[1]
+          #          puts "Where: #{raw_where_clause}"
+          raw_where_clause.split.each do |identifier|
+            next if non_matches.include? identifier
+            # Go through the where clause to find columns that were queried
+            if identifier.include?('.') # e.g., "post"."user_id"
               current_table, column_candidate = identifier.split('.')
-            else
-              current_table, column_candidate = [table, identifier]
+              current_table.gsub! table_quote_char, ''
+              column_candidate.gsub! column_quote_char, ''
+              if non_pk_columns_by_table[current_table] && non_pk_columns_by_table[current_table].include?(column_candidate)
+                # We only care about the identifiers that match up to a table and column.
+                # This is a ghetto way to to avoid having to parse SQL (extremely difficult)
+                if queried_columns_by_table.get_in([current_table, column_candidate])
+                  queried_columns_by_table[current_table][column_candidate] += 1
+                else
+                  queried_columns_by_table[current_table] = {column_candidate => 1}
+                end
+                inferred_table_columns_by_raw_where_clause[raw_where_clause] << [current_table, column_candidate]
+              else
+                non_matches << identifier
+              end
             end
-
-            if non_pk_columns_by_table[current_table].include? column_candidate
-              # We only care about the identifiers that match up to a table and column.
-              # This is a ghetto way to to avoid having to parse SQL (extremely difficult)
-              memo << [current_table, column_candidate]
-            else
-              memo
-            end
-          end.each do |(table, column)|
-            queried_columns_by_table[table] << column
-            inferred_table_columns_by_raw_where_clause[raw_where_clause] << [table, column]
           end
         end
       end
@@ -145,11 +194,15 @@ module SuggestDbIndices
     end
 
     def hash_of_arrays
-      Hash.new {|h, k| h[k] = [] }
+      Hash.new { |h, k| h[k] = [] }
+    end
+
+    def hash_of_hashes
+      Hash.new { |h, k| h[k] = Hash.new }
     end
 
     def hash_of_sets
-      Hash.new {|h, k| h[k] = Set.new }
+      Hash.new { |h, k| h[k] = Set.new }
     end
 
     def remove_limit_clause s
@@ -160,12 +213,6 @@ module SuggestDbIndices
       end
     end
 
-    def strip_color_codes! file_name, output_path
-      # From: http://serverfault.com/a/154200
-      sh_dbg 'sed "s/${esc}[^m]*m//g" ' + "#{file_name} >> #{output_path}"
-      raise "There was a problem stripping colors" unless $?.success?
-    end
-
     def sh_dbg cmd
       puts "Shelling:   #{cmd}"
       `#{cmd}`

data/lib/suggest_db_indices/version.rb

@@ -1,3 +1,3 @@
 module SuggestDbIndices
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/suggest_db_indices.rb

@@ -2,4 +2,5 @@ require 'tempfile'
 require 'rails/generators'
 
 require File.join File.dirname(__FILE__), '/suggest_db_indices/version'
+require File.join File.dirname(__FILE__), '/suggest_db_indices/clojurian_imperialism'
 require File.join File.dirname(__FILE__), '/suggest_db_indices/core'

data/suggest_db_indices.gemspec

@@ -2,19 +2,21 @@
 require File.join File.dirname(__FILE__), 'lib/suggest_db_indices/version'
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Isak Sky"]
-  gem.email         = ["isak.sky@gmail.com"]
-  gem.description   = %q{A gem for your rails project that suggests indices for you to add in your database. Currently it suggests adding indices to unindexed foreign key columns.}
-  gem.summary       = %q{A gem for your rails project that suggests indices for you to add in your database.}
-  gem.homepage      = "https://github.com/isaksky/suggest-db-indices"
-  gem.license       = "MIT"
+  gem.authors = ["Isak Sky"]
+  gem.email = ["isak.sky@gmail.com"]
+  gem.description = %q{A gem for your rails project that suggests indices for you to add in your database. Currently it suggests adding indices to unindexed foreign key columns.}
+  gem.summary = %q{A gem for your rails project that suggests indices for you to add in your database.}
+  gem.homepage = "https://github.com/isaksky/suggest-db-indices"
+  gem.license = "MIT"
 
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "suggest-db-indices"
+  gem.files = `git ls-files`.split($\)
+  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name = "suggest-db-indices"
   gem.require_paths = ["lib"]
-  gem.version       = SuggestDbIndices::VERSION
+  gem.version = SuggestDbIndices::VERSION
 
   gem.add_dependency('rails', '>= 3.0.0')
+  gem.add_development_dependency('awesome_print')
+  gem.add_development_dependency('pry')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: suggest-db-indices
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-11 00:00:00.000000000 Z
+date: 2012-07-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
-  requirement: &2152362540 !ruby/object:Gem::Requirement
+  requirement: &2161081920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,29 @@ dependencies:
         version: 3.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152362540
+  version_requirements: *2161081920
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: &2161081300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2161081300
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: &2161080600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2161080600
 description: A gem for your rails project that suggests indices for you to add in
   your database. Currently it suggests adding indices to unindexed foreign key columns.
 email:
@@ -36,6 +58,7 @@ files:
 - README.md
 - Rakefile
 - lib/suggest_db_indices.rb
+- lib/suggest_db_indices/clojurian_imperialism.rb
 - lib/suggest_db_indices/core.rb
 - lib/suggest_db_indices/version.rb
 - suggest_db_indices.gemspec