full_table_scan_matchers 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 461eb2bcbdf13c351343b98640ee0e110fcd2ff6
+  data.tar.gz: 0fc519dd6462d5befb9cbe595882d50c4850cebc
+SHA512:
+  metadata.gz: e7379fddc8b2b57e61fa4200ae2a005d7be85f8836dfba1f4c75f8bb3612da8a8d635e7ce7710d6b444a649806356ffd82a66e1dff7697beba4a1a2dc77daecd
+  data.tar.gz: 14f459f90f4ed869c77d3620f30aa22e0d80cc9a088feebaeb6e1364eb5881d5434764da724953cf8665a965ae3fa8a35cbbd5abb6263e3c56d1765528c419f9

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+config/database.yml
+
+coverage/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,27 @@
+services:
+  - mysql
+sudo: false
+cache: bundler
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.0
+  - 2.2.0
+  - 2.3.0
+  - 2.3.1
+env:
+  - RAILS_VERSION=4
+  - RAILS_VERSION=5
+matrix:
+  exclude:
+  - rvm: 2.0.0
+    env: RAILS_VERSION=5
+  - rvm: 2.1.0
+    env: RAILS_VERSION=5
+  - rvm: 2.2.0
+    env: RAILS_VERSION=5
+before_install: gem install bundler -v 1.12.5
+before_script:
+  - cp config/database.travis.yml config/database.yml
+  - bin/setup
+script: bundle exec rspec

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in full_table_scan_matchers.gemspec
+gemspec
+
+if ENV['RAILS_VERSION']
+  gem 'activesupport', "~> #{ENV['RAILS_VERSION']}"
+  gem 'activerecord',  "~> #{ENV['RAILS_VERSION']}"
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Justin Aiken
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,88 @@
+[![Gem Version](http://img.shields.io/gem/v/full_table_scan_matchers.svg)](https://rubygems.org/gems/full_table_scan_matchers)[![Build Status](http://img.shields.io/travis/JustinAiken/full_table_scan_matchers/master.svg)](http://travis-ci.org/JustinAiken/full_table_scan_matchers)[![Coveralls branch](http://img.shields.io/coveralls/JustinAiken/full_table_scan_matchers/master.svg)](https://coveralls.io/r/JustinAiken/full_table_scan_matchers?branch=master)
+
+# full_table_scan_matchers
+
+Detect full table scans in your unit tests!  As opposed to just doing basic regexes on the `db/schema.rb` like [rails-best-practices](https://github.com/railsbp/rails_best_practices) does, this actually checks queries made against the database, to ensure and assert that there are indexes for it to use.
+
+## Requirements/Support
+
+- Ruby 2.0+
+- ActiveRecord 4+
+- ActiveSupport 4+
+- mysql2 adapter
+
+## Installation
+
+Add to your `Gemfile`, in the test group:
+
+```ruby
+group :test do
+  gem 'full_table_scan_matchers', github: 'JustinAiken/full_table_scan_matchers'
+end
+```
+
+## Usage
+
+```ruby
+# No full table scans at all please:
+expect { user.posts }.not_to full_table_scan
+
+# Or on just a specific thing:
+expect {
+  user.posts.joins(:comments).first.comments
+}.not_to full_table_scan.on :posts
+```
+
+## How's it work?
+
+- It logs all SQL queries made inside the `expect` block
+  - ..optionally filtering them to table(s) you care about
+- Afterwards, it runs all those through `EXPLAIN`
+- Instead of checking `type` for `ALL`, it checks the keys instead
+  - Because in test mode, mysql may ignore the index and scan the whole table because it's so small
+  - But your production database is much, much larger - we just want to make sure that indexes are available
+
+## Configuration
+
+You can optionally configure a few things:
+
+```ruby
+FullTableScanMatchers.configure do |config|
+  # Tables to ignore - defaults to none
+  # Default: none
+  config.ignores = []
+
+  # Database adapter to use - only one for now
+  # Default (and only): mysql
+  config.adapter = FullTableScanMatchers::DBAdapters::MySql
+
+  # Includes a backtrace in the fail output if this is set to true
+  # Default: false
+  config.log_backtrace = false
+
+  # Add a proc to strip things from the logged backtraces
+  # Default: None
+  config.backtrace_filter = Proc.new { |backtrace| backtrace }
+end
+```
+
+## Postgres?
+
+Currently, this just support mySql.  However, I tried to keep the part that's most-tied to that database (parsing the `EXPLAIN` output) silo'd in an adapter - I'd happily merge any clean PRs that add Postgres (or others)!
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests.
+
+## Credits
+
+- Primary author: [@JustinAiken](https://github.com/JustinAiken)
+- A **lot** of inspiration comes from the excellent [db-query-matchers](https://github.com/brigade/db-query-matchers) by [@brigade](https://github.com/brigade)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/JustinAiken/full_table_scan_matchers.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,31 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
+
+require 'active_record'
+require 'yaml'
+
+namespace :db do
+  db_config       = YAML::load(File.open('config/database.yml'))['test']
+  db_config_admin = db_config.merge('database' => 'mysql')
+
+  desc "Create the database"
+  task :create do
+    ActiveRecord::Base.establish_connection(db_config_admin)
+    ActiveRecord::Base.connection.create_database(db_config["database"])
+    puts "Database created."
+  end
+
+  desc "Drop the database"
+  task :drop do
+    ActiveRecord::Base.establish_connection(db_config_admin)
+    ActiveRecord::Base.connection.drop_database(db_config["database"])
+    puts "Database deleted."
+  end
+
+  desc "Reset the database"
+  task reset: [:drop, :create]
+end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+bundle exec rake db:drop
+bundle exec rake db:create

data/config/database.travis.yml

@@ -0,0 +1,5 @@
+test:
+  adapter: mysql2
+  database: full_table_scan_matchers_test
+  username: travis
+  encoding: utf8

data/full_table_scan_matchers.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'full_table_scan_matchers/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "full_table_scan_matchers"
+  spec.version       = FullTableScanMatchers::VERSION
+  spec.authors       = ["Justin Aiken"]
+  spec.email         = ["60tonangel@gmail.com"]
+
+  spec.summary       = %q{Matchers for rspec to detect full table scans}
+  spec.description   = %q{Matchers for rspec to detect full table scans}
+  spec.homepage      = "https://github.com/JustinAiken/full_table_scan_matchers"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^spec/}) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport", ">= 3.0"
+  spec.add_dependency "activerecord",  ">= 3.0"
+
+  spec.add_development_dependency 'mysql2'
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake",    "~> 10.0"
+  spec.add_development_dependency "rspec",   "~> 3.0"
+  spec.add_development_dependency 'coveralls'
+end

data/lib/full_table_scan_matchers.rb

@@ -0,0 +1,39 @@
+require 'full_table_scan_matchers/version'
+require 'full_table_scan_matchers/db_adapters'
+require 'full_table_scan_matchers/db_adapters/mysql'
+require 'full_table_scan_matchers/db_adapters/mysql/explain_result'
+require 'full_table_scan_matchers/sql_watcher'
+require 'full_table_scan_matchers/matchers/full_table_scan'
+require 'full_table_scan_matchers/configuration'
+
+require 'active_record'
+require 'active_support'
+
+# Main module that holds global configuration.
+module FullTableScanMatchers
+  class << self
+    attr_writer :configuration
+  end
+
+  # Gets the current configuration
+  # @return [FullTableScanMatchers::Configuration] the active configuration
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  # Resets the current configuration.
+  # @return [FullTableScanMatchers::Configuration] the active configuration
+  def self.reset_configuration
+    @configuration = Configuration.new
+  end
+
+  # Updates the current configuration.
+  # @example
+  #   FullTableScanMatchers.configure do |config|
+  #     config.ignores = [/SELECT.*FROM.*users/]
+  #   end
+  #
+  def self.configure
+    yield configuration
+  end
+end

data/lib/full_table_scan_matchers/configuration.rb

@@ -0,0 +1,14 @@
+module FullTableScanMatchers
+
+  # Configuration for the FullTableScanMatchers module.
+  class Configuration
+    attr_accessor :ignores, :adapter, :log_backtrace, :backtrace_filter
+
+    def initialize
+      @ignores          = []
+      @adapter          = DBAdapters::MySql
+      @log_backtrace    = false
+      @backtrace_filter = Proc.new { |backtrace| backtrace }
+    end
+  end
+end

data/lib/full_table_scan_matchers/db_adapters.rb

@@ -0,0 +1,4 @@
+module FullTableScanMatchers
+  module DBAdapters
+  end
+end

data/lib/full_table_scan_matchers/db_adapters/mysql.rb

@@ -0,0 +1,6 @@
+module FullTableScanMatchers
+  module DBAdapters
+    module MySql
+    end
+  end
+end

data/lib/full_table_scan_matchers/db_adapters/mysql/explain_result.rb

@@ -0,0 +1,61 @@
+require 'ostruct'
+
+module FullTableScanMatchers
+  module DBAdapters
+    module MySql
+      class ExplainResult
+        attr_accessor :sql_statement, :structs, :backtrace, :tables
+
+        def initialize(sql_statement, backtrace: nil, tables: nil)
+          @sql_statement = sql_statement
+          @backtrace     = backtrace
+          @tables        = tables
+        end
+
+        def full_table_scan?
+          offending_structs.any?
+        end
+
+        def structs
+          @structs ||= as_hashes.map { |hash| OpenStruct.new hash }
+        end
+
+        def to_s
+          [
+            "SQL:  #{sql_statement}",
+            structs.map do |struct|
+              (offending_structs.include?(struct) ? "FAIL: " : "INFO: ") +
+              struct.to_h.map { |k, v| "#{k}: #{v}" }.join(", ")
+            end,
+            (backtrace.present? ? "BACKTRACE: #{backtrace}" : nil)
+          ].flatten.compact.join "\n"
+        end
+
+      private
+
+        def offending_structs
+          structs
+            .reject  { |struct| struct.table == "NULL" }
+            .select  { |struct| tables.nil? || Array(tables).map(&:to_s).include?(struct.table) }
+            .select  { |struct| struct.key == "NULL" && struct.possible_keys == "NULL" }
+        end
+
+        def as_hashes
+          keys = explain_rows.shift.map(&:downcase)
+          explain_rows.map { |values| Hash[keys.zip(values)].symbolize_keys! }
+        end
+
+        def explain_rows
+          @explain_rows ||= explained_result
+            .reject { |row| row =~ /^\s*\+/ }        # Reject table frames, like: +----+-------------+
+            .reject { |row| row =~ / rows? in set/ } # Reject "1 row in set" type things after result
+            .map    { |row| row.split("|").map(&:strip).reject &:blank? }
+        end
+
+        def explained_result
+          @explained_result ||= ActiveRecord::Base.connection.explain(sql_statement).split("\n")
+        end
+      end
+    end
+  end
+end

data/lib/full_table_scan_matchers/matchers/full_table_scan.rb

@@ -0,0 +1,79 @@
+require 'rspec/core'
+require 'rspec/expectations'
+require 'rspec/mocks'
+
+RSpec::Matchers.define :full_table_scan do |options = {}|
+  if RSpec::Core::Version::STRING =~ /^2/
+    def self.failure_message_when_negated(&block)
+      failure_message_for_should_not(&block)
+    end
+
+    def self.failure_message(&block)
+      failure_message_for_should(&block)
+    end
+
+    def supports_block_expectations?
+      true
+    end
+  else
+    supports_block_expectations
+  end
+
+  # Taken from ActionView::Helpers::TextHelper
+  def pluralize(count, singular, plural = nil)
+    word = if count == 1 || count =~ /^1(\.0+)?$/
+             singular
+           else
+             plural || singular.pluralize
+           end
+
+    "#{count || 0} #{word}"
+  end
+
+  chain :on do |tables|
+    @tables = tables
+  end
+
+  define_method :matches? do |block|
+    watcher_option = {}
+    watcher_option[:tables] = Array(@tables) if defined?(@tables)
+    @watcher = FullTableScanMatchers::SQLWatcher.new(watcher_option)
+    ActiveSupport::Notifications.subscribed @watcher.to_proc, 'sql.active_record', &block
+
+    replay_logged_with_explain!
+
+    @watcher.count > 0
+  end
+
+  def replay_logged_with_explain!
+    @watcher.log
+      .map!    { |logged| FullTableScanMatchers.configuration.adapter::ExplainResult.new(logged[:sql], backtrace: logged[:backtrace], tables: @tables) }
+      .reject! { |logged| !logged.full_table_scan? }
+  end
+
+  def output_offenders
+    @watcher.log.map(&:to_s).join("\n")
+  end
+
+  failure_message_when_negated do |_|
+    <<-EOS
+expected no full table scans, but #{@watcher.log.count} were made:
+#{output_offenders}
+    EOS
+  end
+
+  failure_message do |_|
+    if options[:count]
+      expected = pluralize(options[:count], 'full table scan')
+      actual   = pluralize(@watcher.count, 'was', 'were')
+
+      output   = "expected #{expected}, but #{actual} made"
+      if @watcher.count > 0
+        output += ":\n#{output_offenders}"
+      end
+      output
+    else
+      'expected full table scans, but none were made'
+    end
+  end
+end

data/lib/full_table_scan_matchers/sql_watcher.rb

@@ -0,0 +1,63 @@
+module FullTableScanMatchers
+  class SQLWatcher
+    attr_reader :log, :options
+
+    def initialize(options = {})
+      @log     = []
+      @options = options
+    end
+
+    # Turns a SQLWatcher instance into a lambda. Designed to be used when
+    # subscribing to events through the ActiveSupport::Notifications module.
+    #
+    # @return [Proc]
+    def to_proc
+      lambda &method(:callback)
+    end
+
+    # Method called from the ActiveSupport::Notifications module (through the
+    # lambda created by `to_proc`) when an SQL query is made.
+    #
+    # @param _name       [String] name of the event
+    # @param _start      [Time]   when the instrumented block started execution
+    # @param _finish     [Time]   when the instrumented block ended execution
+    # @param _message_id [String] unique ID for this notification
+    # @param payload     [Hash]   the payload
+    #
+    def callback(_name, _start,  _finish, _message_id, payload)
+      sql_statement = payload[:sql]
+      return if     sql_statement =~ /EXPLAIN /i # That's from us, don't EXPLAIN the EXPLAINS!
+      return unless sql_statement =~ /SELECT /   # Only selects for now
+      return if     any_match? ignores, sql_statement
+      return unless any_match? tables,  sql_statement if options[:tables]
+
+      backtrace = if FullTableScanMatchers.configuration.log_backtrace
+        raw_backtrace      = caller
+        filtered_backtrace = FullTableScanMatchers.configuration.backtrace_filter.call(raw_backtrace)
+        "#{filtered_backtrace.join("\n")}\n"
+      else
+        nil
+      end
+
+      @log << {sql: sql_statement.strip, backtrace: backtrace}
+    end
+
+    def count
+      log.count
+    end
+
+  private
+
+    def tables
+      Array(options[:tables]).map { |table_name| /(FROM|JOIN)\s+`?#{table_name.to_s}`?/i }
+    end
+
+    def ignores
+      FullTableScanMatchers.configuration.ignores
+    end
+
+    def any_match?(patterns, sql)
+      patterns.any? { |pattern| sql =~ pattern }
+    end
+  end
+end

data/lib/full_table_scan_matchers/version.rb

@@ -0,0 +1,3 @@
+module FullTableScanMatchers
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: full_table_scan_matchers
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Justin Aiken
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Matchers for rspec to detect full table scans
+email:
+- 60tonangel@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/setup
+- config/database.travis.yml
+- full_table_scan_matchers.gemspec
+- lib/full_table_scan_matchers.rb
+- lib/full_table_scan_matchers/configuration.rb
+- lib/full_table_scan_matchers/db_adapters.rb
+- lib/full_table_scan_matchers/db_adapters/mysql.rb
+- lib/full_table_scan_matchers/db_adapters/mysql/explain_result.rb
+- lib/full_table_scan_matchers/matchers/full_table_scan.rb
+- lib/full_table_scan_matchers/sql_watcher.rb
+- lib/full_table_scan_matchers/version.rb
+homepage: https://github.com/JustinAiken/full_table_scan_matchers
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.6
+signing_key: 
+specification_version: 4
+summary: Matchers for rspec to detect full table scans
+test_files: []