oink 0.1.2 → 0.9.1

data/lib/oink/middleware.rb

@@ -0,0 +1,48 @@
+require 'hodel_3000_compliant_logger'
+require 'oink/utils/hash_utils'
+require 'oink/instrumentation'
+
+module Oink
+  class Middleware
+
+    def initialize(app, logpath = "log/oink.log")
+      ActiveRecord::Base.send(:include, Oink::Instrumentation::ActiveRecord)
+      @logger = Hodel3000CompliantLogger.new(logpath)
+      @app = app
+    end
+
+    def call(env)
+      status, headers, body = @app.call(env)
+      log_routing_information(env)
+      @logger.info("Completed in")
+      log_memory_snapshot
+      log_objects_instantiated
+      reset_objects_instantiated
+      [status, headers, body]
+    end
+
+    def log_routing_information(env)
+      if env.has_key?('action_dispatch.request.parameters')
+        controller = env['action_dispatch.request.parameters']['controller']
+        action = env['action_dispatch.request.parameters']['action']
+        @logger.info "Processing #{controller}##{action}"
+      end
+    end
+
+    def log_memory_snapshot
+      memory = Oink::Instrumentation::MemorySnapshot.memory
+      @logger.info("Memory usage: #{memory} | PID: #{$$}")
+    end
+
+    def log_objects_instantiated
+      sorted_list = Oink::HashUtils.to_sorted_array(ActiveRecord::Base.instantiated_hash)
+      sorted_list.unshift("Total: #{ActiveRecord::Base.total_objects_instantiated}")
+      @logger.info("Instantiation Breakdown: #{sorted_list.join(' | ')}")
+    end
+
+    def reset_objects_instantiated
+      ActiveRecord::Base.reset_instance_type_count
+    end
+
+  end
+end

data/lib/oink/rails/instance_type_counter.rb

@@ -1,19 +1,14 @@
-module Oink
-
-  def self.extended_active_record?
-    @oink_extended_active_record
-  end
+require 'oink/instrumentation/active_record'
+require 'oink/utils/hash_utils'
 
-  def self.extended_active_record!
-    @oink_extended_active_record = true
-  end
+module Oink
 
   module InstanceTypeCounter
     def self.included(klass)
-      ActiveRecord::Base.send(:include, OinkInstanceTypeCounterInstanceMethods)
+      ActiveRecord::Base.send(:include, Oink::Instrumentation::ActiveRecord)
 
       klass.class_eval do
-        after_filter :report_instance_type_count
+        around_filter :report_instance_type_count
       end
     end
 
@@ -23,8 +18,9 @@ module Oink
     private
 
       def report_instance_type_count
-        report_hash = ActiveRecord::Base.instantiated_hash.merge("Total" => ActiveRecord::Base.total_objects_instantiated)
-        breakdown = report_hash.sort{|a,b| b[1]<=>a[1]}.collect {|k,v| "#{k}: #{v}" }.join(" | ")
+        sorted_list = Oink::HashUtils.to_sorted_array(ActiveRecord::Base.instantiated_hash)
+        sorted_list.unshift("Total: #{ActiveRecord::Base.total_objects_instantiated}")
+        breakdown = sorted_list.join(" | ")
         before_report_active_record_count(breakdown)
         if logger
           logger.info("Instantiation Breakdown: #{breakdown}")
@@ -33,54 +29,4 @@ module Oink
       end
   end
 
-  module OinkInstanceTypeCounterInstanceMethods
-
-    def self.included(klass)
-      klass.class_eval do
-
-        @@instantiated = {}
-        @@total = nil
-
-        def self.reset_instance_type_count
-          @@instantiated = {}
-          @@total = nil
-        end
-
-        def self.increment_instance_type_count
-          @@instantiated[base_class.name] ||= 0
-          @@instantiated[base_class.name] += 1
-        end
-
-        def self.instantiated_hash
-          @@instantiated
-        end
-
-        def self.total_objects_instantiated
-          @@total ||= @@instantiated.values.sum
-        end
-
-        unless Oink.extended_active_record?
-          class << self
-            alias_method :allocate_before_oink, :allocate
-
-            def allocate
-              value = allocate_before_oink
-              increment_instance_type_count
-              value
-            end
-          end
-
-          alias_method :initialize_before_oink, :initialize
-
-          def initialize(*args, &block)
-            value = initialize_before_oink(*args, &block)
-            self.class.increment_instance_type_count
-            value
-          end
-
-          Oink.extended_active_record!
-        end
-      end
-    end
-  end
 end

data/lib/oink/rails/memory_usage_logger.rb

@@ -1,47 +1,25 @@
-begin
-  require 'win32ole'
-rescue LoadError
-end
+require 'oink/instrumentation/memory_snapshot'
 
 module Oink
   module MemoryUsageLogger
     def self.included(klass)
       klass.class_eval do
-        after_filter :log_memory_usage
+        around_filter :log_memory_usage
       end
     end
-  
-    private
-      def get_memory_usage
-        if defined? WIN32OLE
-          wmi = WIN32OLE.connect("winmgmts:root/cimv2")
-          mem = 0
-          query = "select * from Win32_Process where ProcessID = #{$$}"
-          wmi.ExecQuery(query).each do |wproc|
-            mem = wproc.WorkingSetSize
-          end
-          mem.to_i / 1000
-        elsif pages = File.read("/proc/self/statm") rescue nil
-          pages.to_i * statm_page_size
-        elsif proc_file = File.new("/proc/#{$$}/smaps") rescue nil
-          proc_file.map do |line|
-            size = line[/Size: *(\d+)/, 1] and size.to_i
-          end.compact.sum
-        else
-          `ps -o vsz= -p #{$$}`.to_i
-        end
-      end
 
-      # try to get and cache memory page size. falls back to 4096.
-      def statm_page_size
-        @statm_page_size ||= (`getconf PAGESIZE`.strip.to_i rescue 4096) / 1024
-      end
+  private
 
-      def log_memory_usage
-        if logger
-          memory_usage = get_memory_usage
+    def log_memory_usage
+      yield
+      if logger
+        begin
+          memory_usage = Instrumentation::MemorySnapshot.memory
           logger.info("Memory usage: #{memory_usage} | PID: #{$$}")
+        rescue Oink::Instrumentation::MemoryDataUnavailableError => e
+          logger.error("Oink unable to retrieve memory on this system. See Oink::MemorySnapshot in source.")
         end
       end
+    end
   end
 end

data/lib/oink/reports/active_record_instantiation_oinked_request.rb

@@ -0,0 +1,13 @@
+require "oink/reports/request"
+
+module Oink
+  module Reports
+    class ActiveRecordInstantiationOinkedRequest < Request
+
+      def display_oink_number
+        @oink_number
+      end
+
+    end
+  end
+end

data/lib/oink/reports/active_record_instantiation_report.rb

@@ -0,0 +1,67 @@
+require "date"
+require "oink/reports/base"
+require "oink/reports/active_record_instantiation_oinked_request"
+require "oink/reports/priority_queue"
+
+module Oink
+  module Reports
+    class ActiveRecordInstantiationReport < Base
+
+      def print(output)
+        output.puts "---- OINK FOR ACTIVERECORD ----"
+        output.puts "THRESHOLD: #{@threshold} Active Record objects per request\n"
+
+        output.puts "\n-- REQUESTS --\n" if @format == :verbose
+
+        @inputs.each do |input|
+          input.each_line do |line|
+            line = line.strip
+
+             # Skip this line since we're only interested in the Hodel 3000 compliant lines
+            next unless line =~ HODEL_LOG_FORMAT_REGEX
+
+            if line =~ /rails\[(\d+)\]/
+              pid = $1
+              @pids[pid] ||= { :buffer => [], :ar_count => -1, :action => "", :request_finished => true }
+              @pids[pid][:buffer] << line
+            end
+
+            if line =~ /Processing ((\w+)#(\w+)) /
+
+              @pids[pid][:action] = $1
+              unless @pids[pid][:request_finished]
+                @pids[pid][:buffer] = [line]
+              end
+              @pids[pid][:request_finished] = false
+
+            elsif line =~ /Instantiation Breakdown: Total: (\d+)/
+
+              @pids[pid][:ar_count] = $1.to_i
+
+            elsif line =~ /Completed in/
+
+              if @pids[pid][:ar_count] > @threshold
+                @bad_actions[@pids[pid][:action]] ||= 0
+                @bad_actions[@pids[pid][:action]] = @bad_actions[@pids[pid][:action]] + 1
+                date = HODEL_LOG_FORMAT_REGEX.match(line).captures[0]
+                @bad_requests.push(ActiveRecordInstantiationOinkedRequest.new(@pids[pid][:action], date, @pids[pid][:buffer], @pids[pid][:ar_count]))
+                if @format == :verbose
+                  @pids[pid][:buffer].each { |b| output.puts b }
+                  output.puts "---------------------------------------------------------------------"
+                end
+              end
+
+              @pids[pid][:request_finished] = true
+              @pids[pid][:buffer] = []
+              @pids[pid][:ar_count] = -1
+
+            end # end elsif
+          end # end each_line
+        end # end each input
+
+        print_summary(output)
+
+      end
+    end
+  end
+end

data/lib/oink/reports/base.rb

@@ -0,0 +1,38 @@
+module Oink
+  module Reports
+    class Base
+
+      FORMATS = %w[verbose short-summary summary]
+      FORMAT_ALIASES = { "v" => "verbose", "ss" => "short-summary", "s" => "summary" }
+      HODEL_LOG_FORMAT_REGEX = /^(\w+ \d{2} \d{2}:\d{2}:\d{2})/
+
+      def initialize(input, threshold, options = {})
+        @inputs = Array(input)
+        @threshold = threshold
+        @format = options[:format] || :short_summary
+
+        @pids = {}
+        @bad_actions = {}
+        @bad_requests = PriorityQueue.new(10)
+      end
+
+    protected
+
+      def print_summary(output)
+        output.puts "\n-- SUMMARY --\n"
+        output.puts "Worst Requests:"
+        @bad_requests.each_with_index do |offender, index|
+          output.puts "#{index + 1}. #{offender.datetime}, #{offender.display_oink_number}, #{offender.action}"
+          if @format == :summary
+            offender.log_lines.each { |b| output.puts b }
+            output.puts "---------------------------------------------------------------------"
+          end
+        end
+        output.puts "\nWorst Actions:"
+        @bad_actions.sort{|a,b| b[1]<=>a[1]}.each { |elem|
+          output.puts "#{elem[1]}, #{elem[0]}"
+        }
+      end
+    end
+  end
+end

data/lib/oink/reports/memory_oinked_request.rb

@@ -0,0 +1,13 @@
+require "oink/reports/request"
+
+module Oink
+  module Reports
+    class MemoryOinkedRequest < Request
+
+      def display_oink_number
+        "#{@oink_number} KB"
+      end
+
+    end
+  end
+end

data/lib/oink/reports/memory_usage_report.rb

@@ -0,0 +1,71 @@
+require "date"
+require "oink/reports/base"
+require "oink/reports/memory_oinked_request"
+require "oink/reports/priority_queue"
+
+module Oink
+  module Reports
+    class MemoryUsageReport < Base
+      def print(output)
+        output.puts "---- MEMORY THRESHOLD ----"
+        output.puts "THRESHOLD: #{@threshold/1024} MB\n"
+
+        output.puts "\n-- REQUESTS --\n" if @format == :verbose
+
+        @inputs.each do |input|
+          input.each_line do |line|
+            line = line.strip
+
+             # Skip this line since we're only interested in the Hodel 3000 compliant lines
+            next unless line =~ HODEL_LOG_FORMAT_REGEX
+
+            if line =~ /rails\[(\d+)\]/
+              pid = $1
+              @pids[pid] ||= { :buffer => [], :last_memory_reading => -1, :current_memory_reading => -1, :action => "", :request_finished => true }
+              @pids[pid][:buffer] << line
+            end
+
+            if line =~ /Processing ((\w+)#(\w+)) /
+
+              unless @pids[pid][:request_finished]
+                @pids[pid][:last_memory_reading] = -1
+              end
+              @pids[pid][:action] = $1
+              @pids[pid][:request_finished] = false
+
+            elsif line =~ /Memory usage: (\d+) /
+
+              memory_reading = $1.to_i
+              @pids[pid][:current_memory_reading] = memory_reading
+
+            elsif line =~ /Completed in/
+
+              @pids[pid][:request_finished] = true
+              unless @pids[pid][:current_memory_reading] == -1 || @pids[pid][:last_memory_reading] == -1
+                memory_diff = @pids[pid][:current_memory_reading] - @pids[pid][:last_memory_reading]
+                if memory_diff > @threshold
+                  @bad_actions[@pids[pid][:action]] ||= 0
+                  @bad_actions[@pids[pid][:action]] = @bad_actions[@pids[pid][:action]] + 1
+                  date = HODEL_LOG_FORMAT_REGEX.match(line).captures[0]
+                  @bad_requests.push(MemoryOinkedRequest.new(@pids[pid][:action], date, @pids[pid][:buffer], memory_diff))
+                  if @format == :verbose
+                    @pids[pid][:buffer].each { |b| output.puts b }
+                    output.puts "---------------------------------------------------------------------"
+                  end
+                end
+              end
+
+              @pids[pid][:buffer] = []
+              @pids[pid][:last_memory_reading] = @pids[pid][:current_memory_reading]
+              @pids[pid][:current_memory_reading] = -1
+
+            end # end elsif
+          end # end each_line
+        end # end each input
+
+        print_summary(output)
+
+      end
+    end
+  end
+end

data/lib/oink/reports/priority_queue.rb

@@ -0,0 +1,41 @@
+module Oink
+  module Reports
+    class PriorityQueue
+
+      include Enumerable
+
+      def initialize(size)
+        @size = size
+        @queue = []
+      end
+
+      def push(item)
+        if @queue.size < @size
+          @queue << item
+        elsif item > @queue.last
+          @queue[@size - 1] = item
+        end
+        prioritize
+      end
+
+      def to_a
+        @queue
+      end
+
+      def size
+        @queue.size
+      end
+
+      def each
+        @queue.each { |i| yield i }
+      end
+
+    protected
+
+      def prioritize
+        @queue.sort! { |a, b| b <=> a }
+      end
+
+    end
+  end
+end