active_record_scanner 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c72aa49167eea88d5833e44a2b3b4a7ccf73235f
+  data.tar.gz: 0369e6f6fbb7512c42df9b0f58b239025f77e503
+SHA512:
+  metadata.gz: cbb013cf5922c304fe3a397fcfdf81dbdb3822f5c4e639c975df88ff6e05acacc41fd76bdfc65278983d1a1995ead0670cfd039b2c5044d9b03337536a55bd13
+  data.tar.gz: ce78065b3b0739fa714301840694d18b2dc0cebba739487011e2a2f4daff439411b8748fccd3a71c0abee91d8c0f8e8d2b6d92db357bc394d723f1d39dbfbd1d

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+.byebug_history
+.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--color

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in active_record_scanner.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,35 @@
+PATH
+  remote: .
+  specs:
+    active_record_scanner (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  active_record_scanner!
+  bundler (~> 1.16)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Sean Goedecke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# Active Record Scanner 
+
+A static analysis tool that detects known ORM performance issues with ActiveRecord (e.g. queries inside inner loops).
+
+## Installation
+
+    $ gem install active_record_scanner
+
+## Usage
+
+`active_record_scanner "/full/path/to/file/**/*.rb` will recursively scan a directory for ActiveRecord queries inside loops. Consider replacing with methods that operate on collections: for instance, replace `find_by` inside a loop with a `where` call.
+
+Example:
+```
+sgoedecke:parser/ (master*) $ active_record_scanner ./spec/fixtures/test_class.rb 
+.
+./spec/fixtures/test_class.rb (line 24 column 4) -- called query method '#destroy' in a loop
+```
+
+## Development
+
+Run the tests with `rake`.
+
+## References
+
+This tool was inspired by this paper by Junwen Yang: https://newtraell.cs.uchicago.edu/files/ms_paper/junwen.pdf
+
+Unlike the static analysis tools described in the paper, this is (a) written in Ruby and (b) not reliant on a series of regexes.
+
+## Todo
+
+* Improve sexp tree traversal (e.g. avoid mutating the tree in `normalize!`)
+* Improve nested array compacting. Remove that awful loop
+* Add checks for inefficiences other than queries inside loops
+* Add a homebrew formula for easier use
+
+Contributions are welcome.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/active_record_scanner.gemspec

@@ -0,0 +1,26 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "active_record_scanner/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_record_scanner"
+  spec.version       = ActiveRecordScanner::VERSION
+  spec.authors       = ["Sean Goedecke"]
+  spec.email         = ["sean.goedecke@gmail.com"]
+
+  spec.summary       = %q{Scan your Rails project for inefficient AR queries}
+  spec.homepage      = "https://github.com/sgoedecke/active_record_scanner"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/bin/active_record_scanner

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require 'active_record_scanner'
+require 'optionparser'
+
+options = {}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: ruby scanner.rb [options] [full-path]"
+  opts.on("-s", "--silent", "Suppress dot reporting in output") do |_s|
+    options[:silent] = true
+  end
+end
+option_parser.parse!
+
+if ARGV[0]
+  results = ActiveRecordScanner::Scanner.new(ARGV[0], options).scan
+  puts "\r\n" unless options[:silent]
+  puts results.join("\n")
+else
+  puts option_parser.help 
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "active_record_scanner"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_record_scanner/compactor.rb

@@ -0,0 +1,33 @@
+class Compactor 
+  # This is really bad. It works for now but I'd really like to figure out a better way.
+  def deep_compact(n)
+    10.times do # won't work if we get a tree with >10 depth
+      n = deep_compact_with_nesting(strip_nesting(n))
+    end
+    n
+  end
+
+  private
+  
+  def deep_compact_with_nesting(n)
+    if n.respond_to? :compact
+      n.compact.map{ |c| deep_compact_with_nesting(c) }
+    else
+      n
+    end
+  end
+
+  def strip_nesting(n)
+    if n.is_a?(Array)
+      if n.length == 0
+        return nil
+      elsif n.length == 1
+        return strip_nesting(n.first)
+      else
+        return n.map{ |n| strip_nesting(n) }
+      end
+    else
+      return n
+    end
+  end
+end

data/lib/active_record_scanner/constants.rb

@@ -0,0 +1,41 @@
+AR_METHODS = %w(
+find
+find_by
+find_by!
+exists?
+destroy
+delete
+create_with
+distinct
+eager_load
+extending
+from
+group
+having
+includes
+joins
+left_outer_joins
+limit
+lock
+none
+offset
+order
+preload
+readonly
+references
+reorder
+reverse_order
+select
+where
+)
+
+LOOP_METHODS = %w(
+map
+each
+times
+for_each
+flat_map
+each_with_index
+reduce
+)
+

data/lib/active_record_scanner/parser.rb

@@ -0,0 +1,65 @@
+require 'ripper'
+require 'active_record_scanner/compactor'
+
+class Parser
+  def initialize(raw)
+    @raw = raw
+    @compactor = Compactor.new
+  end
+
+  def parse
+    raw_tree = Ripper.sexp(@raw)
+    return [] unless raw_tree # if the file isn't valid ruby, ignore it
+    filtered_tree = filter(raw_tree)
+    compact_tree = @compactor.deep_compact(filtered_tree) 
+    return [] unless compact_tree # if the file has no queries or loops, ignore it
+    normalise!(compact_tree)
+  end
+
+  private
+
+  def is_block?(node)
+    [:brace_block, :do_block].include?(node[0])
+  end
+
+  def is_query?(node)
+    node[0] == :@ident && AR_METHODS.include?(node[1])
+  end
+
+  def is_loop?(node)
+    node[0] == :@ident && LOOP_METHODS.include?(node[1])
+  end
+
+  # walk the tree and return a new tree with all nodes that aren't
+  # loops, blocks or queries turned to `nil`
+  def filter(new_tree = [], tree)
+    tree.map.with_index do |node, i|
+      if node.is_a?(Array) # we only care about non-leaf nodes
+        if is_loop?(node)
+          [ :loop, node, nil ]
+        elsif is_block?(node)
+          [ :block, node, filter(node)]
+        elsif is_query?(node)
+          # if the current node is a query, return it
+          [:query, node] 
+        elsif node
+          filter(node)
+        end
+      end
+    end
+  end
+
+  # walk the tree looking for blocks that immediately follow calls to
+  # loop methods, and tag these blocks as blocks that are run inside loops
+  def normalise!(tree)
+    tree.each.with_index do |node, i|
+      if node.is_a?(Array)
+        if node.first == :loop
+          tree[i+1][0] = :lblock if tree[i+1]
+        end
+        tree[i] = normalise!(node)
+      end
+    end
+    tree
+  end
+end

data/lib/active_record_scanner/version.rb

@@ -0,0 +1,3 @@
+module ActiveRecordScanner
+  VERSION = "0.1.0"
+end

data/lib/active_record_scanner.rb

@@ -0,0 +1,50 @@
+require "active_record_scanner/version"
+require "active_record_scanner/constants"
+require "active_record_scanner/parser"
+
+module ActiveRecordScanner
+  class Scanner 
+    def initialize(glob, options={})
+      @glob = glob
+      @options = options
+      @results = []
+    end
+
+    def scan
+      Dir.glob(@glob).each do |file|
+        scan_file(file)
+      end
+      @results
+    end
+
+    private
+
+    def scan_file(file)
+      print "." unless @options[:silent]
+      @file = file
+      raw = IO.binread(file)
+      compact_tree = Parser.new(raw).parse
+      scan_for_errors(compact_tree)
+    end
+
+    def report_error(error)
+      node = error.last
+      @results << "#{@file} (line #{node[2][1]} column #{node[2][0]}) -- called query method '##{node[1]}' in a loop"
+    end
+
+    def scan_for_errors(node, inside_loop = false)
+      return unless node.is_a?(Array) # don't traverse into raw values
+
+      # if there's a loop higher up in the tree, set the flag
+      # TODO: turn this into a counter to flag queries inside inner loops as more serious
+      inside_loop = true if node[0] == :lblock
+
+      report_error(node) if node[0] == :query && inside_loop
+
+      node.each do |child|
+        scan_for_errors(child, inside_loop)
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: active_record_scanner
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sean Goedecke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-10-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email:
+- sean.goedecke@gmail.com
+executables:
+- active_record_scanner
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- active_record_scanner.gemspec
+- bin/active_record_scanner
+- bin/console
+- bin/setup
+- lib/active_record_scanner.rb
+- lib/active_record_scanner/compactor.rb
+- lib/active_record_scanner/constants.rb
+- lib/active_record_scanner/parser.rb
+- lib/active_record_scanner/version.rb
+homepage: https://github.com/sgoedecke/active_record_scanner
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.4
+signing_key: 
+specification_version: 4
+summary: Scan your Rails project for inefficient AR queries
+test_files: []