active-record-profiler 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 29a3517f08a6a21ab80d888178ae5371f60f9f05
+  data.tar.gz: 752933b5e705af4c7698496246f2356df6007dbb
+SHA512:
+  metadata.gz: e23f7711b5a0936b5d55e14027f3befd06bf7f6d999d3a4e8b8efcc636d0b41e78cd69cef3f4d11d3c7ba9a673ad50bf2e829a1ea58172e5580ff8a8d09ce74a
+  data.tar.gz: 3099240d90f3497e8f2f95597e5489c4b487d58e0fc234f29190d9ad4ca3daefe125ffecf05a73b8ac9d8588b1b247eb8c69091eb2fd6683a10c9ce2ad869e13

data/.gitignore

@@ -0,0 +1,3 @@
+test/dummy/log/*
+test/dummy/db/*.sqlite3
+test/dummy/tmp/*

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# 0.1.0
+
+* Convert from plugin to gem
+
+TODO: Convert to an engine for the reports

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in static-data.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2010 Gist, Inc.
+Copyright (c) 2015 Benjamin Turner
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,150 @@
+ActiveRecordProfiler
+====================
+
+ActiveRecordProfiler monkey-patches 
+ActiveRecord::ConnectionAdapters::AbstractAdapter both to improve the 
+standard SQL logging and to provide profiler-like tracking of SQL
+statements generated by application code.
+
+Each SQL log entry generated by ActiveRecord will have appended the 
+filename, line number, and function name of the nearest calling 
+application code. The information is determined by walking up the call
+stack until a filename within the /app/ directory is found. If no such
+filename can be found, the SQL will be logged with a location of 
+'Non-application code'.
+
+Additionally, the profiler will keep track of the total time spent by all
+SQL calls coming from each calling location, as well as the number of time
+that location accessed the database. Certain SQL calls not under the
+direct control of the application are not counted in these statistics,
+such as "SHOW FIELDS", "SET NAMES", "BEGIN", and "COMMIT" statements which
+tend to skew the timing statistics and provide less useful information
+about slow SQL queries.
+
+Periodically, the profiler will dump its statistics out to a file and
+restart all of its counters/timers. The output file is named for the time
+and PID from which it was written, so that multiple processes can safely
+write their output simultaneously.
+
+Installation
+============
+Add it to your Gemfile, do `bundle install`, and then add a new initializer, `config/initializers/active_record_profiler.rb`:
+
+    ActiveRecord::Base.logger =
+        ActiveRecordProfiler::Logger.new(ActiveRecord::Base.logger)
+    ActiveRecordProfiler::LogSubscriber.attach_to :active_record unless Rails.env.test?
+
+
+The first line adds call-site information to ActiveRecord logging, and the second line enables profiling (except in the test environment, where it would mess up your profiling data).
+
+Configuration
+=============
+Control the (approximate) frequency of statistics flushes (default: `1.hour`)
+
+    ActiveRecordProfiler::Collector.stats_flush_period = 1.hour
+
+Note that only flushed data is available for use in the rake reports (described below). If you are running a multithreaded or multiprocess server (which covers most common rails server types), your data will be incomplete until all those threads/processes/servers have flushed their data. This limitation exists primarily to avoid the overhead of coordinating/locking during the process of serving your application's web requests.
+
+Directory where profile data is recorded (default: `Rails.root,join('log', 'profiler_data'`)
+
+    ActiveRecordProfiler::Collector.profile_dir = Rails.root.join('log', 'profiler_data'
+
+Any SQL statements matching this pattern will not be tracked by the 
+profiler output, though it will still appear in the enhanced SQL logging
+(default: `/^(SHOW FIELDS |SET SQL_AUTO_IS_NULL|SET NAMES |EXPLAIN |BEGIN|COMMIT|PRAGMA )/`)
+
+    ActiveRecordProfiler::Collector.sql_ignore_pattern = /^SET /x
+
+
+Reports
+=======
+To see a top-100 list of what SQL statements your application is spending its
+time in, run the following rake task:
+
+    rake profiler:aggregate max_lines=100 show_sql=true
+
+This will return a list of the SQL which is taking the most time in your 
+application in this format:
+
+    <file path>:<line number>:in <method name>: <total duration>, <call count>, <max single call duration>
+
+This will aggregate all of the profiler data you have accumulated; in order 
+to limit the timeframe of the data, use the `prefix` option to specify a
+partial date/time:
+
+    rake profiler:aggregate max_lines=100 show_sql=true prefix=2010-06-20-10  # data from June 20 during the 10am hour (roughly)
+
+Each thread running the profiler flushes its stats periodically, and there
+is a rake task to combine multiple profiler data files together in order to 
+keep the number of data files down to a manageable number. A good way to 
+manage the data files on a server is to set up a cron task to run the 
+following command once per hour or once per day:
+
+    rake profiler:aggregate compact=<'hour' or 'date'> RAILS_ENV=qa
+
+Compacting by hour will result in a single file for each hour any process 
+dumped its stats. Compacting by day will result in a single file for each 
+day. When using the `prefix` option to generate a profiler report, you
+cannot specify an hour if you have compacted your data by date instead of
+hour (the prefix matching operates on the file names, which will not have
+hours if they have been compacted by date).
+
+You can clear out all profiler data using the following command:
+
+    rake profiler:clear_data
+  
+If you want programmatic access to the profiler data, check out the source
+code for the rake tasks in `lib/active-record-profiler/tasks.rake`.
+
+
+=======
+HTML Reports
+============
+
+The profiler includes some view helpers to make it easy for your application
+to generate a sortable HTML table of profiler information. The core helper
+method generates a table based on an ActiveRecordProfiler::Collector object.
+In its simplest form, it can be called from a view like this:
+
+  <div id="#profiler">
+    <%= profiler_report(params) %>
+  </div>
+  
+The full set of parameters available looks like this:
+
+  profiler_report(page_parameters, date_prefix, sort_id, max_rows, profiler_collector, css_options)
+  
+parameters:
+  page_parameters: this is generally the request parameters, used to build the report-sorting links (HashWithIndifferentAccess)
+  options: hash containing optional settings for the report; supported keys:
+  
+    :date : year, year-month, year-month-day, year-month-day-hour used to filter the profiler data; defaults to Today (String)
+    :sort : ActiveRecordProfiler::(DURATION|COUNT|LONGEST|AVG_DURATION) specifying which field to sort the report by; defaults to DURATION (Constant/Integer)
+    :max_rows : Maximum number of table rows to output; in other words, report on the top max_rows SQL statements; defaults to 100 (Integer)
+    :collector : object representing the profile data to use in building the report; defaults to an empty collector using the configured profile data directory (ActiveRecordProfiler::Collector )
+    :table : css class applied to the report <table> element; defaults to nil
+    :header_row : css class applied to the report's header row; defaults to nil
+    :row : css class applied to the report's data rows; defaults to nil
+    :link_locations : true/false value indicating whether to build textmate links to the source code whence a given piece of SQL came; defaults to false
+  
+An easy way to support filtering of report data by month/date/hour is to 
+use a view like this:
+
+  <%= profiler_date_filter_form(params) %>
+  <%= profiler_report(params) %>
+  
+And if you use TextMate, then you may want to throw in some extra goodies
+to generate links to the actual source code files and lines where the SQL
+was triggered (Note: the current javascript requires jQuery):
+
+  <%= profiler_date_filter_form(params) %>
+  <%= profiler_report_local_path_form %>
+  <%= profile_report_local_path_javascript %>
+  <%= profiler_report(params, {:link_locations => true}) %>
+
+
+Miscellaneous
+=============
+
+Copyright (c) 2010 Gist, Inc.
+Copyright (c) 2015 Benjamin Turner

data/Rakefile

@@ -0,0 +1,14 @@
+require "bundler"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+desc "Run tests"
+task :default => :test
+
+Bundler::GemHelper.install_tasks

data/active-record-profiler.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'active-record-profiler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "active-record-profiler"
+  spec.version       = ActiveRecordProfiler::VERSION
+  spec.authors       = ["Ben Turner"]
+  spec.email         = ["codewrangler@outofcoffee.com"]
+  spec.summary       = %q{Enhances ActiveRecord logging and profiles queries}
+  spec.description   = <<-EOF
+See where each database call is coming from in your code, and get query
+profiling to see which queries are taking up the most time in the database.
+EOF
+  spec.homepage      = "https://github.com/kindjar/active_record_profiler"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rails", "~> 4.0"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake", "~> 10.3"
+  spec.add_development_dependency "minitest", "~> 5.1"
+  spec.add_development_dependency "sqlite3", "~> 1.3"
+end

data/lib/active-record-profiler.rb

@@ -0,0 +1,13 @@
+require 'active-record-profiler/version'
+require 'active-record-profiler/collector'
+require 'active-record-profiler/log_subscriber'
+require 'active-record-profiler/logger'
+require 'active-record-profiler/profiler_view_helper'
+
+require 'active-record-profiler/railtie' if defined?(Rails)
+
+module ActiveRecordProfiler
+  require 'fileutils'
+  require 'json'
+end
+

data/lib/active-record-profiler/collector.rb

@@ -0,0 +1,276 @@
+module ActiveRecordProfiler
+  class Collector
+    DURATION = 0
+    COUNT = 1
+    LONGEST = 2
+    LONGEST_SQL = 3
+    LOCATION = -1
+    AVG_DURATION = -2
+    
+    DATE_FORMAT = '%Y-%m-%d'
+    HOUR_FORMAT = '-%H'
+    DATETIME_FORMAT = DATE_FORMAT + HOUR_FORMAT + '-%M'
+    AGGREGATE_QUIET_PERIOD = 1.minutes
+    
+    CSV_DURATION = 0
+    CSV_COUNT = 1
+    CSV_AVG = 2
+    CSV_LONGEST = 3
+    CSV_LOCATION = 4
+    CSV_LONGEST_SQL = 5
+
+    NON_APP_CODE_DESCRIPTION = 'Non-application code'
+         
+    cattr_accessor :profiler_enabled
+    self.profiler_enabled = true
+    
+    # You can control the (approximate) frequency of statistics flushes by setting
+    # ActiveRecordProfiler::Collector.stats_flush_period = 1.hour
+    cattr_accessor :stats_flush_period
+    self.stats_flush_period = 1.hour
+    
+    # Directory where profile data is recorded
+    cattr_accessor :profile_dir
+
+    # Any SQL statements matching this pattern will not be tracked by the profiler output
+    # (though it will still appear in the enhanced SQL logging).
+    cattr_accessor :sql_ignore_pattern
+    self.sql_ignore_pattern = /^(SHOW FIELDS |SET SQL_AUTO_IS_NULL|SET NAMES |EXPLAIN |BEGIN|COMMIT|PRAGMA )/i
+    
+    cattr_accessor :app_path_pattern
+    self.app_path_pattern = Regexp.new(Regexp.quote("/app/"))
+    
+    cattr_accessor :trim_root_path
+    
+    cattr_accessor :storage_backend
+    self.storage_backend = :json
+    
+    cattr_accessor :profile_self
+    self.profile_self = false
+
+    attr_accessor :last_stats_flush
+    attr_accessor :query_sites
+    attr_accessor :profile_data_directory
+    
+    def self.instance
+      Thread.current[:active_record_profiler_collector] ||= Collector.new
+    end
+    
+    def self.profile_self?
+      self.profile_self
+    end
+    
+    def self.clear_data
+      dir = Dir.new(profile_dir)
+      prof_files = dir.entries.select{ |filename| /.prof$/.match(filename) }.map{ |filename| File.join(dir.path, filename) }
+      FileUtils.rm(prof_files) if prof_files.size > 0
+    end
+
+    def initialize
+      @query_sites = {}
+      @last_stats_flush = nil
+      @profile_data_directory = self.class.profile_dir
+    end
+    
+    def call_location_name(caller_array = nil)
+      find_app_call_location(caller_array) || NON_APP_CODE_DESCRIPTION
+    end
+    
+    def record_caller_info(location, seconds, sql)
+      return if sql_ignore_pattern.match(sql)
+      
+      update_counts(location, seconds, 1, sql)
+    end
+    
+    def record_self_info(seconds, name)
+      record_caller_info(trim_location(caller.first), seconds, name)
+    end
+    
+    def should_flush_stats?
+      self.last_stats_flush ||= Time.now
+      return(Time.now > self.last_stats_flush + stats_flush_period)
+    end
+    
+    def flush_query_sites_statistics
+      pid = $$
+      thread_id = Thread.current.object_id
+      flush_time = Time.now
+      site_count = self.query_sites.keys.size
+      Rails.logger.info("Flushing ActiveRecordProfiler statistics for PID #{pid} at #{flush_time} (#{site_count} sites).")
+      
+      if (site_count > 0)
+        FileUtils.makedirs(self.profile_data_directory)
+      
+        filename = File.join(self.profile_data_directory, "#{flush_time.strftime(DATETIME_FORMAT)}.#{pid}-#{thread_id}.prof")
+        write_file(filename)
+        
+        # Nuke each value to make sure it can be reclaimed by Ruby
+        self.query_sites.keys.each{ |k| self.query_sites[k] = nil }
+      end
+      self.query_sites = {}
+      self.last_stats_flush = flush_time
+    end
+    
+    def aggregate(options = {})
+      prefix = options[:prefix]
+      compact = options[:compact]
+      raise "Cannot compact without a prefix!" if compact && prefix.nil?
+      return self.query_sites unless File.exists?(self.profile_data_directory)
+      
+      dir = Dir.new(self.profile_data_directory)
+      now = Time.now
+      raw_files_processed = []
+      date_regexp = Regexp.new(prefix) if prefix
+
+      dir.each do |filename|
+        next unless /.prof$/.match(filename)
+        next if date_regexp && ! date_regexp.match(filename)
+        # Parse the datetime out of the filename and convert it to localtime
+        begin
+          file_time = DateTime.strptime(filename, DATETIME_FORMAT)
+          file_time = Time.local(file_time.year, file_time.month, file_time.day, file_time.hour, file_time.min)
+        rescue Exception => e
+          if e.to_s != 'invalid date'
+            raise e
+          end
+        end
+
+        if (file_time.nil? || ((file_time + AGGREGATE_QUIET_PERIOD) < now))
+          begin
+            update_from_file(File.join(dir.path, filename))
+          
+            raw_files_processed << filename if file_time    # any files that are already aggregated don't count
+          rescue Exception => e
+            RAILS_DEFAULT_LOGGER.warn "Unable to read file #{filename}: #{e.message}"
+          end
+        else
+          Rails.logger.info "Skipping file #{filename} because it is too new and may still be open for writing."
+        end
+      end
+
+      if compact && raw_files_processed.size > 0
+        write_file(File.join(dir.path, "#{prefix}.prof"))
+
+        raw_files_processed.each do |filename|
+          FileUtils.rm(File.join(dir.path, filename))
+        end
+      end
+
+      return self.query_sites
+    end
+
+    def save_aggregated(date = nil)
+      aggregate(:date => date, :compact => true)
+    end
+
+    def sorted_locations(sort_field = nil, max_locations = nil)
+      sort_field ||= DURATION
+      case sort_field
+        when LOCATION
+          sorted = self.query_sites.keys.sort
+        when AVG_DURATION
+          sorted = self.query_sites.keys.sort_by{ |k| (self.query_sites[k][DURATION] / self.query_sites[k][COUNT]) }.reverse
+        when DURATION, COUNT, LONGEST
+          sorted = self.query_sites.keys.sort{ |a,b| self.query_sites[b][sort_field] <=> self.query_sites[a][sort_field] }
+        else
+          raise "Invalid sort field: #{sort_field}"
+      end
+      if max_locations && max_locations > 0
+        sorted.first(max_locations)
+      else
+        sorted
+      end
+    end
+
+    protected
+
+    def find_app_call_location(call_stack)
+      call_stack = caller
+      while frame = call_stack.shift
+        if app_path_pattern.match(frame)
+          return trim_location(frame)
+        end
+      end
+      return nil
+    end
+    
+    def trim_location(loc)
+      loc.sub(trim_root_path, '')
+    end
+    
+    def update_counts(location, seconds, count, sql, longest = nil)
+      longest ||= seconds
+      self.query_sites[location] ||= [0.0,0,0,'']
+      self.query_sites[location][DURATION] += seconds
+      self.query_sites[location][COUNT] += count
+      if (longest > self.query_sites[location][LONGEST])
+        self.query_sites[location][LONGEST] = longest
+        self.query_sites[location][LONGEST_SQL] = sql.to_s
+      end
+    end
+    
+    def detect_file_type(filename)
+      type = nil
+      File.open(filename, "r") do |io|
+        first_line = io.readline
+        if first_line.match(/^\/\* JSON \*\//)
+          type = :json
+        end
+      end
+      return type
+    end
+    
+    def write_file(filename)
+      case storage_backend
+      when :json
+        write_json_file(filename)
+      else
+        raise "Invalid storage_backend: #{storage_backend}"
+      end
+    end
+    
+    def write_json_file(filename)
+      File.open(filename, "w") do |file|
+        file.puts "/* JSON */"
+        file.puts "/* Fields: Duration, Count, Avg. Duration, Max. Duration, Location, Max. Duration SQL */"
+        file.puts "["
+
+        first = true
+        self.query_sites.each_pair do |location, info|
+          if first
+            first = false
+          else
+            file.puts "\n, "
+          end
+          row = [info[DURATION], info[COUNT], (info[DURATION]/info[COUNT]), info[LONGEST], location, info[LONGEST_SQL]]
+          file.print JSON.generate(row)
+        end
+        file.puts "\n]"
+      end
+    end
+    
+    def update_from_file(filename)
+      read_file(filename) do |row|
+        update_counts(
+          row[CSV_LOCATION], row[CSV_DURATION].to_f, row[CSV_COUNT].to_i, row[CSV_LONGEST_SQL], row[CSV_LONGEST].to_f
+        )
+      end
+    end
+    
+    def read_file(filename)
+      file_type = detect_file_type filename
+      case file_type
+      when :json
+        read_json_file(filename) { |row| yield row }
+      else
+        raise "Unknown profiler data file type for file '#{filename}: #{file_type}"
+      end
+    end
+    
+    def read_json_file(filename)
+      JSON.load(File.open(filename, "r")).each do |row|
+        yield row
+      end
+    end
+  end
+end