oink 0.1.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/README.rdoc +15 -29
  2. data/Rakefile +4 -6
  3. data/bin/oink +2 -2
  4. data/lib/oink.rb +1 -9
  5. data/lib/oink/cli.rb +71 -67
  6. data/lib/oink/instrumentation.rb +2 -0
  7. data/lib/oink/instrumentation/active_record.rb +63 -0
  8. data/lib/oink/instrumentation/memory_snapshot.rb +119 -0
  9. data/lib/oink/middleware.rb +48 -0
  10. data/lib/oink/rails/instance_type_counter.rb +8 -62
  11. data/lib/oink/rails/memory_usage_logger.rb +11 -33
  12. data/lib/oink/reports/active_record_instantiation_oinked_request.rb +13 -0
  13. data/lib/oink/reports/active_record_instantiation_report.rb +67 -0
  14. data/lib/oink/reports/base.rb +38 -0
  15. data/lib/oink/reports/memory_oinked_request.rb +13 -0
  16. data/lib/oink/reports/memory_usage_report.rb +71 -0
  17. data/lib/oink/reports/priority_queue.rb +41 -0
  18. data/lib/oink/reports/request.rb +20 -0
  19. data/lib/oink/utils/hash_utils.rb +9 -0
  20. data/spec/fakes/fake_application_controller.rb +30 -0
  21. data/spec/fakes/psuedo_output.rb +7 -0
  22. data/spec/helpers/database.rb +20 -0
  23. data/spec/{rails → oink/instrumentation}/instance_type_counter_spec.rb +11 -9
  24. data/spec/oink/instrumentation/memory_snapshot_spec.rb +84 -0
  25. data/spec/oink/middleware_spec.rb +73 -0
  26. data/spec/oink/rails/instance_type_counter_spec.rb +52 -0
  27. data/spec/oink/rails/memory_usage_logger_spec.rb +23 -0
  28. data/spec/oink/reports/active_record_instantiation_report_spec.rb +193 -0
  29. data/spec/oink/reports/memory_usage_report_spec.rb +267 -0
  30. data/spec/oink/reports/oinked_request_spec.rb +22 -0
  31. data/spec/oink/reports/priority_queue_spec.rb +74 -0
  32. data/spec/spec_helper.rb +10 -26
  33. metadata +158 -29
  34. data/lib/oink/active_record_instantiation_reporter.rb +0 -68
  35. data/lib/oink/base.rb +0 -40
  36. data/lib/oink/memory_usage_reporter.rb +0 -72
  37. data/lib/oink/oinked_request/oinked_ar_request.rb +0 -9
  38. data/lib/oink/oinked_request/oinked_memory_request.rb +0 -9
  39. data/lib/oink/oinked_request/oinked_request.rb +0 -16
  40. data/lib/oink/priority_queue.rb +0 -37
  41. data/spec/oink/active_record_instantiation_reporter_spec.rb +0 -191
  42. data/spec/oink/memory_usage_reporter_spec.rb +0 -265
  43. data/spec/oinked_request/oinked_request_spec.rb +0 -20
  44. data/spec/priority_queue/priority_queue_spec.rb +0 -75
  45. data/spec/rails/memory_usage_logger_spec.rb +0 -87
@@ -0,0 +1,48 @@
1
+ require 'hodel_3000_compliant_logger'
2
+ require 'oink/utils/hash_utils'
3
+ require 'oink/instrumentation'
4
+
5
+ module Oink
6
+ class Middleware
7
+
8
+ def initialize(app, logpath = "log/oink.log")
9
+ ActiveRecord::Base.send(:include, Oink::Instrumentation::ActiveRecord)
10
+ @logger = Hodel3000CompliantLogger.new(logpath)
11
+ @app = app
12
+ end
13
+
14
+ def call(env)
15
+ status, headers, body = @app.call(env)
16
+ log_routing_information(env)
17
+ @logger.info("Completed in")
18
+ log_memory_snapshot
19
+ log_objects_instantiated
20
+ reset_objects_instantiated
21
+ [status, headers, body]
22
+ end
23
+
24
+ def log_routing_information(env)
25
+ if env.has_key?('action_dispatch.request.parameters')
26
+ controller = env['action_dispatch.request.parameters']['controller']
27
+ action = env['action_dispatch.request.parameters']['action']
28
+ @logger.info "Processing #{controller}##{action}"
29
+ end
30
+ end
31
+
32
+ def log_memory_snapshot
33
+ memory = Oink::Instrumentation::MemorySnapshot.memory
34
+ @logger.info("Memory usage: #{memory} | PID: #{$$}")
35
+ end
36
+
37
+ def log_objects_instantiated
38
+ sorted_list = Oink::HashUtils.to_sorted_array(ActiveRecord::Base.instantiated_hash)
39
+ sorted_list.unshift("Total: #{ActiveRecord::Base.total_objects_instantiated}")
40
+ @logger.info("Instantiation Breakdown: #{sorted_list.join(' | ')}")
41
+ end
42
+
43
+ def reset_objects_instantiated
44
+ ActiveRecord::Base.reset_instance_type_count
45
+ end
46
+
47
+ end
48
+ end
@@ -1,19 +1,14 @@
1
- module Oink
2
-
3
- def self.extended_active_record?
4
- @oink_extended_active_record
5
- end
1
+ require 'oink/instrumentation/active_record'
2
+ require 'oink/utils/hash_utils'
6
3
 
7
- def self.extended_active_record!
8
- @oink_extended_active_record = true
9
- end
4
+ module Oink
10
5
 
11
6
  module InstanceTypeCounter
12
7
  def self.included(klass)
13
- ActiveRecord::Base.send(:include, OinkInstanceTypeCounterInstanceMethods)
8
+ ActiveRecord::Base.send(:include, Oink::Instrumentation::ActiveRecord)
14
9
 
15
10
  klass.class_eval do
16
- after_filter :report_instance_type_count
11
+ around_filter :report_instance_type_count
17
12
  end
18
13
  end
19
14
 
@@ -23,8 +18,9 @@ module Oink
23
18
  private
24
19
 
25
20
  def report_instance_type_count
26
- report_hash = ActiveRecord::Base.instantiated_hash.merge("Total" => ActiveRecord::Base.total_objects_instantiated)
27
- breakdown = report_hash.sort{|a,b| b[1]<=>a[1]}.collect {|k,v| "#{k}: #{v}" }.join(" | ")
21
+ sorted_list = Oink::HashUtils.to_sorted_array(ActiveRecord::Base.instantiated_hash)
22
+ sorted_list.unshift("Total: #{ActiveRecord::Base.total_objects_instantiated}")
23
+ breakdown = sorted_list.join(" | ")
28
24
  before_report_active_record_count(breakdown)
29
25
  if logger
30
26
  logger.info("Instantiation Breakdown: #{breakdown}")
@@ -33,54 +29,4 @@ module Oink
33
29
  end
34
30
  end
35
31
 
36
- module OinkInstanceTypeCounterInstanceMethods
37
-
38
- def self.included(klass)
39
- klass.class_eval do
40
-
41
- @@instantiated = {}
42
- @@total = nil
43
-
44
- def self.reset_instance_type_count
45
- @@instantiated = {}
46
- @@total = nil
47
- end
48
-
49
- def self.increment_instance_type_count
50
- @@instantiated[base_class.name] ||= 0
51
- @@instantiated[base_class.name] += 1
52
- end
53
-
54
- def self.instantiated_hash
55
- @@instantiated
56
- end
57
-
58
- def self.total_objects_instantiated
59
- @@total ||= @@instantiated.values.sum
60
- end
61
-
62
- unless Oink.extended_active_record?
63
- class << self
64
- alias_method :allocate_before_oink, :allocate
65
-
66
- def allocate
67
- value = allocate_before_oink
68
- increment_instance_type_count
69
- value
70
- end
71
- end
72
-
73
- alias_method :initialize_before_oink, :initialize
74
-
75
- def initialize(*args, &block)
76
- value = initialize_before_oink(*args, &block)
77
- self.class.increment_instance_type_count
78
- value
79
- end
80
-
81
- Oink.extended_active_record!
82
- end
83
- end
84
- end
85
- end
86
32
  end
@@ -1,47 +1,25 @@
1
- begin
2
- require 'win32ole'
3
- rescue LoadError
4
- end
1
+ require 'oink/instrumentation/memory_snapshot'
5
2
 
6
3
  module Oink
7
4
  module MemoryUsageLogger
8
5
  def self.included(klass)
9
6
  klass.class_eval do
10
- after_filter :log_memory_usage
7
+ around_filter :log_memory_usage
11
8
  end
12
9
  end
13
-
14
- private
15
- def get_memory_usage
16
- if defined? WIN32OLE
17
- wmi = WIN32OLE.connect("winmgmts:root/cimv2")
18
- mem = 0
19
- query = "select * from Win32_Process where ProcessID = #{$$}"
20
- wmi.ExecQuery(query).each do |wproc|
21
- mem = wproc.WorkingSetSize
22
- end
23
- mem.to_i / 1000
24
- elsif pages = File.read("/proc/self/statm") rescue nil
25
- pages.to_i * statm_page_size
26
- elsif proc_file = File.new("/proc/#{$$}/smaps") rescue nil
27
- proc_file.map do |line|
28
- size = line[/Size: *(\d+)/, 1] and size.to_i
29
- end.compact.sum
30
- else
31
- `ps -o vsz= -p #{$$}`.to_i
32
- end
33
- end
34
10
 
35
- # try to get and cache memory page size. falls back to 4096.
36
- def statm_page_size
37
- @statm_page_size ||= (`getconf PAGESIZE`.strip.to_i rescue 4096) / 1024
38
- end
11
+ private
39
12
 
40
- def log_memory_usage
41
- if logger
42
- memory_usage = get_memory_usage
13
+ def log_memory_usage
14
+ yield
15
+ if logger
16
+ begin
17
+ memory_usage = Instrumentation::MemorySnapshot.memory
43
18
  logger.info("Memory usage: #{memory_usage} | PID: #{$$}")
19
+ rescue Oink::Instrumentation::MemoryDataUnavailableError => e
20
+ logger.error("Oink unable to retrieve memory on this system. See Oink::MemorySnapshot in source.")
44
21
  end
45
22
  end
23
+ end
46
24
  end
47
25
  end
@@ -0,0 +1,13 @@
1
+ require "oink/reports/request"
2
+
3
+ module Oink
4
+ module Reports
5
+ class ActiveRecordInstantiationOinkedRequest < Request
6
+
7
+ def display_oink_number
8
+ @oink_number
9
+ end
10
+
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,67 @@
1
+ require "date"
2
+ require "oink/reports/base"
3
+ require "oink/reports/active_record_instantiation_oinked_request"
4
+ require "oink/reports/priority_queue"
5
+
6
+ module Oink
7
+ module Reports
8
+ class ActiveRecordInstantiationReport < Base
9
+
10
+ def print(output)
11
+ output.puts "---- OINK FOR ACTIVERECORD ----"
12
+ output.puts "THRESHOLD: #{@threshold} Active Record objects per request\n"
13
+
14
+ output.puts "\n-- REQUESTS --\n" if @format == :verbose
15
+
16
+ @inputs.each do |input|
17
+ input.each_line do |line|
18
+ line = line.strip
19
+
20
+ # Skip this line since we're only interested in the Hodel 3000 compliant lines
21
+ next unless line =~ HODEL_LOG_FORMAT_REGEX
22
+
23
+ if line =~ /rails\[(\d+)\]/
24
+ pid = $1
25
+ @pids[pid] ||= { :buffer => [], :ar_count => -1, :action => "", :request_finished => true }
26
+ @pids[pid][:buffer] << line
27
+ end
28
+
29
+ if line =~ /Processing ((\w+)#(\w+)) /
30
+
31
+ @pids[pid][:action] = $1
32
+ unless @pids[pid][:request_finished]
33
+ @pids[pid][:buffer] = [line]
34
+ end
35
+ @pids[pid][:request_finished] = false
36
+
37
+ elsif line =~ /Instantiation Breakdown: Total: (\d+)/
38
+
39
+ @pids[pid][:ar_count] = $1.to_i
40
+
41
+ elsif line =~ /Completed in/
42
+
43
+ if @pids[pid][:ar_count] > @threshold
44
+ @bad_actions[@pids[pid][:action]] ||= 0
45
+ @bad_actions[@pids[pid][:action]] = @bad_actions[@pids[pid][:action]] + 1
46
+ date = HODEL_LOG_FORMAT_REGEX.match(line).captures[0]
47
+ @bad_requests.push(ActiveRecordInstantiationOinkedRequest.new(@pids[pid][:action], date, @pids[pid][:buffer], @pids[pid][:ar_count]))
48
+ if @format == :verbose
49
+ @pids[pid][:buffer].each { |b| output.puts b }
50
+ output.puts "---------------------------------------------------------------------"
51
+ end
52
+ end
53
+
54
+ @pids[pid][:request_finished] = true
55
+ @pids[pid][:buffer] = []
56
+ @pids[pid][:ar_count] = -1
57
+
58
+ end # end elsif
59
+ end # end each_line
60
+ end # end each input
61
+
62
+ print_summary(output)
63
+
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,38 @@
1
+ module Oink
2
+ module Reports
3
+ class Base
4
+
5
+ FORMATS = %w[verbose short-summary summary]
6
+ FORMAT_ALIASES = { "v" => "verbose", "ss" => "short-summary", "s" => "summary" }
7
+ HODEL_LOG_FORMAT_REGEX = /^(\w+ \d{2} \d{2}:\d{2}:\d{2})/
8
+
9
+ def initialize(input, threshold, options = {})
10
+ @inputs = Array(input)
11
+ @threshold = threshold
12
+ @format = options[:format] || :short_summary
13
+
14
+ @pids = {}
15
+ @bad_actions = {}
16
+ @bad_requests = PriorityQueue.new(10)
17
+ end
18
+
19
+ protected
20
+
21
+ def print_summary(output)
22
+ output.puts "\n-- SUMMARY --\n"
23
+ output.puts "Worst Requests:"
24
+ @bad_requests.each_with_index do |offender, index|
25
+ output.puts "#{index + 1}. #{offender.datetime}, #{offender.display_oink_number}, #{offender.action}"
26
+ if @format == :summary
27
+ offender.log_lines.each { |b| output.puts b }
28
+ output.puts "---------------------------------------------------------------------"
29
+ end
30
+ end
31
+ output.puts "\nWorst Actions:"
32
+ @bad_actions.sort{|a,b| b[1]<=>a[1]}.each { |elem|
33
+ output.puts "#{elem[1]}, #{elem[0]}"
34
+ }
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,13 @@
1
+ require "oink/reports/request"
2
+
3
+ module Oink
4
+ module Reports
5
+ class MemoryOinkedRequest < Request
6
+
7
+ def display_oink_number
8
+ "#{@oink_number} KB"
9
+ end
10
+
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,71 @@
1
+ require "date"
2
+ require "oink/reports/base"
3
+ require "oink/reports/memory_oinked_request"
4
+ require "oink/reports/priority_queue"
5
+
6
+ module Oink
7
+ module Reports
8
+ class MemoryUsageReport < Base
9
+ def print(output)
10
+ output.puts "---- MEMORY THRESHOLD ----"
11
+ output.puts "THRESHOLD: #{@threshold/1024} MB\n"
12
+
13
+ output.puts "\n-- REQUESTS --\n" if @format == :verbose
14
+
15
+ @inputs.each do |input|
16
+ input.each_line do |line|
17
+ line = line.strip
18
+
19
+ # Skip this line since we're only interested in the Hodel 3000 compliant lines
20
+ next unless line =~ HODEL_LOG_FORMAT_REGEX
21
+
22
+ if line =~ /rails\[(\d+)\]/
23
+ pid = $1
24
+ @pids[pid] ||= { :buffer => [], :last_memory_reading => -1, :current_memory_reading => -1, :action => "", :request_finished => true }
25
+ @pids[pid][:buffer] << line
26
+ end
27
+
28
+ if line =~ /Processing ((\w+)#(\w+)) /
29
+
30
+ unless @pids[pid][:request_finished]
31
+ @pids[pid][:last_memory_reading] = -1
32
+ end
33
+ @pids[pid][:action] = $1
34
+ @pids[pid][:request_finished] = false
35
+
36
+ elsif line =~ /Memory usage: (\d+) /
37
+
38
+ memory_reading = $1.to_i
39
+ @pids[pid][:current_memory_reading] = memory_reading
40
+
41
+ elsif line =~ /Completed in/
42
+
43
+ @pids[pid][:request_finished] = true
44
+ unless @pids[pid][:current_memory_reading] == -1 || @pids[pid][:last_memory_reading] == -1
45
+ memory_diff = @pids[pid][:current_memory_reading] - @pids[pid][:last_memory_reading]
46
+ if memory_diff > @threshold
47
+ @bad_actions[@pids[pid][:action]] ||= 0
48
+ @bad_actions[@pids[pid][:action]] = @bad_actions[@pids[pid][:action]] + 1
49
+ date = HODEL_LOG_FORMAT_REGEX.match(line).captures[0]
50
+ @bad_requests.push(MemoryOinkedRequest.new(@pids[pid][:action], date, @pids[pid][:buffer], memory_diff))
51
+ if @format == :verbose
52
+ @pids[pid][:buffer].each { |b| output.puts b }
53
+ output.puts "---------------------------------------------------------------------"
54
+ end
55
+ end
56
+ end
57
+
58
+ @pids[pid][:buffer] = []
59
+ @pids[pid][:last_memory_reading] = @pids[pid][:current_memory_reading]
60
+ @pids[pid][:current_memory_reading] = -1
61
+
62
+ end # end elsif
63
+ end # end each_line
64
+ end # end each input
65
+
66
+ print_summary(output)
67
+
68
+ end
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,41 @@
1
+ module Oink
2
+ module Reports
3
+ class PriorityQueue
4
+
5
+ include Enumerable
6
+
7
+ def initialize(size)
8
+ @size = size
9
+ @queue = []
10
+ end
11
+
12
+ def push(item)
13
+ if @queue.size < @size
14
+ @queue << item
15
+ elsif item > @queue.last
16
+ @queue[@size - 1] = item
17
+ end
18
+ prioritize
19
+ end
20
+
21
+ def to_a
22
+ @queue
23
+ end
24
+
25
+ def size
26
+ @queue.size
27
+ end
28
+
29
+ def each
30
+ @queue.each { |i| yield i }
31
+ end
32
+
33
+ protected
34
+
35
+ def prioritize
36
+ @queue.sort! { |a, b| b <=> a }
37
+ end
38
+
39
+ end
40
+ end
41
+ end