lackac-request-log-analyzer 0.1.3

data/output/blockers.rb

@@ -0,0 +1,11 @@
+# Print requests that took more than a second to complete, sorted by their frequency.
+amount = $arguments[:amount] || 10
+
+puts
+puts "Mongrel process blockers (> #{$summarizer.blocker_duration} seconds)"
+puts green("========================================================================")
+
+$summarizer.sort_blockers_by(:count).reverse[0, amount.to_i].each do |a|
+ puts "%-50s: %10.03fs [#{green("%d requests")}]" % [a[0], a[1][:total_time], a[1][:count]]
+end
+

data/output/errors.rb

@@ -0,0 +1,9 @@
+# Print errors that occured often
+amount = $arguments[:amount] || 10
+puts
+puts "Errors"
+puts green("========================================================================")
+$summarizer.sort_errors_by(:count).reverse[0, amount.to_i].each do |a|
+  puts "%s: [#{green("%d requests")}]" % [a[0] + 'Error', a[1][:count]]
+  puts blue(' -> ' + (a[1][:exception_strings].invert[ a[1][:exception_strings].values.max ])[0..79])
+end

data/output/hourly_spread.rb

@@ -0,0 +1,28 @@
+# Draws a graph containing the average amound of requests per hour per day
+if $summarizer.request_time_graph?
+
+  max_request_graph = $summarizer.request_time_graph.max / $summarizer.duration
+  deviation         = max_request_graph / 20
+  deviation         = 1 if deviation == 0
+  color_cutoff      = 15
+  
+  puts
+  puts "Requests graph - average per day per hour"
+  puts green("========================================================================")
+  
+  (0..23).each do |a|
+    requests      = $summarizer.request_time_graph[a] / $summarizer.duration
+    display_chars = requests / deviation
+    
+    if display_chars >= color_cutoff
+      display_chars_string = green(' ΢' * color_cutoff) + red(' ΢' * (display_chars - color_cutoff))
+    else
+      display_chars_string = green(' ΢' * display_chars)
+    end
+    
+    puts "#{a.to_s.rjust(10)}:00 - #{('[' + requests.to_s + ' req.]').ljust(15)} : #{display_chars_string}"
+  end
+else
+  puts
+  puts "Hourly spread not available"
+end

data/output/mean_db_time.rb

@@ -0,0 +1,7 @@
+# Prints a table sorted by the highest database times.
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} worst DB offenders - mean time"
+puts green("========================================================================")
+print_table($summarizer, :mean_db_time, amount)
+

data/output/mean_rendering_time.rb

@@ -0,0 +1,7 @@
+# Prints a table showing the slowest renderes
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} slow renderers - mean time"
+puts green("========================================================================")
+print_table($summarizer, :mean_rendering_time, amount)
+

data/output/mean_time.rb

@@ -0,0 +1,7 @@
+# Prints  table sorted by the duration of the requests
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} actions by time - per request mean"
+puts green("========================================================================")
+
+print_table($summarizer, :mean_time, amount)

data/output/most_requested.rb

@@ -0,0 +1,6 @@
+# Prints a table sorted by the most frequently requested actions
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} most requested actions"
+puts green("========================================================================")
+print_table($summarizer, :count, amount)

data/output/timespan.rb

@@ -0,0 +1,15 @@
+# Prints the total timespan found in the parsed log files.
+puts
+puts green("========================================================================")
+
+if $summarizer.has_timestamps?
+  puts "Timestamp first request: #{$summarizer.first_request_at}"
+  puts "Timestamp last request:  #{$summarizer.last_request_at}" 
+  puts "Total time analyzed: #{$summarizer.duration} days"
+end
+
+methods_print_array = []
+$summarizer.methods.each do |key, value|
+  methods_print_array << "%s (%s)" % [key, green(((value * 100) / $summarizer.request_count).to_s + '%')]
+end
+puts 'Methods: ' + methods_print_array.join(', ') + '.'

data/output/total_db_time.rb

@@ -0,0 +1,6 @@
+# Prints a list of the actions that spend most of their time waiting for database results.
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} worst DB offenders - cumulative time"
+puts green("========================================================================")
+print_table($summarizer, :total_db_time, amount)

data/output/total_time.rb

@@ -0,0 +1,6 @@
+# Prints a list ordered by the requests that took the most time in total.
+amount = $arguments[:amount] || 10
+puts
+puts "Top #{amount} actions by time - cumulative"
+puts green("========================================================================")
+print_table($summarizer, :total_time, amount)

data/output/usage.rb

@@ -0,0 +1,15 @@
+# Prints usage table
+puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
+puts
+puts "Options:"
+puts "  --fast, -t:                 Only use completed requests"
+puts "  --guess-database-time, -g:  Guesses the database duration of requests" 
+puts "  --output, -o:               Comma-separated list of reports to show"     
+puts "  --amount, -c:               Displays the top <amount> elements in the reports"     
+puts "  --colorize, -z:             Fancy bash coloring"
+puts "  --merb, -m:                 Parse merb logfiles"
+puts 
+puts "Examples:"
+puts "  request-log-analyzer development.log"
+puts "  request-log-analyzer mongrel.0.log mongrel.1.log mongrel.2.log -g -f -o mean_time,most_requested,blockers -c 20 -z"
+puts

data/test/log_fragments/fragment_1.log

@@ -0,0 +1,59 @@
+Processing DashboardController#index (for 130.89.162.199 at 2008-08-14 21:16:25) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
+QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
+ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
+bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
+MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
+ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
+MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
+  Parameters: {"action"=>"index", "controller"=>"dashboard"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering dashboard/index
+Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
+
+
+Processing PeopleController#index (for 130.89.162.199 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
+bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
+eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
+bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
+ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
+bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
+  Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
+Set language to: nl_NL
+Redirected to https://www.example.com/people/2545
+Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
+
+
+Processing PeopleController#show (for 130.89.162.199 at 2008-08-14 21:16:30) [GET]
+  Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
+dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
+c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
+EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
+IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
+ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
+bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
+X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
+  Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/show
+person: Gaby Somers, study_year: 2008/2009
+Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
+
+
+Processing PeopleController#picture (for 130.89.162.199 at 2008-08-14 21:16:35) [GET]
+  Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
+SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
+MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
+dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
+YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
+bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
+DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
+  Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
+Set language to: nl_NL
+Rendering template within layouts/priscilla
+Rendering people/picture
+Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]

data/test/log_fragments/fragment_2.log

@@ -0,0 +1,5 @@
+Jul 13 06:25:58 10.1.1.32 app_p [1957]: Processing EmployeeController#index (for 10.1.1.33 at 2008-07-13 06:25:58) [GET]
+Jul 13 06:25:58 10.1.1.32 app_p [1957]: Session ID: bd1810833653be11c38ad1e5675635bd
+Jul 13 06:25:58 10.1.1.32 app_p [1957]: Parameters: {"format"=>"xml", "action"=>"index}
+Jul 13 06:25:58 10.1.1.32 app_p [1957]: Rendering employees
+Jul 13 06:25:58 10.1.1.32 app_p [1957]: Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://example.com/employee.xml]

data/test/log_fragments/merb_1.log

@@ -0,0 +1,84 @@
+~ Loaded DEVELOPMENT Environment...
+~ Connecting to database...
+~ loading gem 'merb_datamapper' ...
+~ loading gem 'gettext' ...
+~ loading gem 'merb_helpers' ...
+~ loading gem 'merb-assets' ...
+~ loading gem 'merb-action-args' ...
+~ loading gem 'merb-mailer' ...
+~ loading gem 'merb_param_protection' ...
+~ loading gem 'merb_has_flash' ...
+~ loading gem 'merb_forgery_protection' ...
+~ loading gem 'dm-validations' ...
+~ loading gem 'dm-timestamps' ...
+~ loading gem 'dm-migrations' ...
+~ loading gem 'dm-aggregates' ...
+~ loading gem 'dm-adjust' ...
+~ loading gem 'dm-serializer' ...
+~ loading gem 'dm-constraints' ...
+~ loading gem 'dm-timeline' ...
+~ loading gem 'dm-searchable' ...
+~ loading gem 'dm-audited' ...
+~ loading gem 'lib/extensions' ...
+~ loading gem 'lib/authenticated_system/authenticated_dependencies' ...
+~ Compiling routes...
+~ Using 'share-nothing' cookie sessions (4kb limit per client)
+~ Using Mongrel adapter
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy", "method"=>"delete", "controller"=>"session"}
+~ Cookie deleted: auth_token => nil
+~ Redirecting to: / (302)
+~ {:dispatch_time=>0.243424, :after_filters_time=>6.9e-05, :before_filters_time=>0.213213, :action_time=>0.241652}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"action"=>"index", "controller"=>"dashboard"}
+~ Redirecting to: /login (302)
+~ {:dispatch_time=>0.002649, :after_filters_time=>7.4e-05, :action_time=>0.001951}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"action"=>"create", "controller"=>"session"}
+~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"authenticity_token"=>"[FILTERED]", "action"=>"create", "controller"=>"session", "login"=>"username", "password"=>"[FILTERED]", "remember_me"=>"0"}
+~ Redirecting to: / (302)
+~ {:dispatch_time=>0.006652, :after_filters_time=>0.000143, :before_filters_time=>0.000861, :action_time=>0.006171}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"action"=>"index", "controller"=>"dashboard"}
+~ Redirecting to: /dashboard (302)
+~ {:dispatch_time=>0.008241, :after_filters_time=>0.000126, :before_filters_time=>0.002632, :action_time=>0.007711}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
+~ {:dispatch_time=>0.009458, :after_filters_time=>0.000103, :before_filters_time=>0.00266, :action_time=>0.008742}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:29 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.102725, :after_filters_time=>0.000115, :before_filters_time=>0.00411, :action_time=>0.101836}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:30 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"organisations"}
+~ {:dispatch_time=>0.042575, :after_filters_time=>8.9e-05, :before_filters_time=>0.004267, :action_time=>0.041762}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:31 +0200 2008
+~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
+~ {:dispatch_time=>0.010311, :after_filters_time=>8.0e-05, :before_filters_time=>0.003195, :action_time=>0.009567}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:33 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.012913, :after_filters_time=>7.1e-05, :before_filters_time=>0.004422, :action_time=>0.012141}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:35 +0200 2008
+~ Params: {"action"=>"new", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.013051, :after_filters_time=>7.8e-05, :before_filters_time=>0.003576, :action_time=>0.011773}

data/test/merb_log_parser_test.rb

@@ -0,0 +1,39 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/merb_analyzer/log_parser"
+
+class MerbLogParserTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/merb_#{number}.log"
+  end
+  
+  def test_parse_started_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:started) do |request|
+      requests << request
+    end
+    assert_equal requests[0][:timestamp], "Fri Aug 29 11:10:23 +0200 2008"
+  end
+
+  def test_parse_completed_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:completed) do |request|
+      requests << request
+    end
+    
+    assert_equal requests[0][:action_time], 0.241652
+  end
+
+  def test_parse_params_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:params) do |request|
+      requests << request
+    end
+    
+    assert_match '"controller"=>"session"', requests[0][:raw_hash]
+    assert_match '"action"=>"destroy"', requests[0][:raw_hash]
+  end
+  
+end

data/test/rails_log_parser_test.rb

@@ -0,0 +1,86 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
+
+class RailsLogParserTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
+  end
+  
+  
+  def test_progress_messages
+    log_file = fragment_file(1)
+
+    finished_encountered = false
+    file_size = File.size(log_file)
+    
+    previous_pos = -1
+    parser = RailsAnalyzer::LogParser.new(log_file)
+    parser.progress do |pos, total|
+      assert_equal file_size, total
+      if pos == :finished
+        finished_encountered = true 
+      else
+        assert pos <= total
+        assert pos > previous_pos
+        previous_pos = pos
+      end
+    end
+    
+    # now parse the file
+    parser.each(:started) { }
+    
+    assert finished_encountered, "A finished event should have been fired"
+  end
+  
+  def test_parse_mongrel_log_fragment
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started) { count += 1 }
+    assert_equal 4, count
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:completed) { count += 1 }
+    assert_equal 4, count    
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started, :completed) { count += 1 }
+    assert_equal 8, count    
+  end
+  
+  def test_parse_syslog_fragment
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) { count += 1 }
+    assert_equal 1, count
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) { count += 1 }
+    assert_equal 1, count    
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started, :completed) { count += 1 }
+    assert_equal 2, count    
+  end
+  
+  def test_parse_syslog_fragment_content
+    # this test only works because there is only one requests in the fragment
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) do |request|
+      assert_equal "EmployeeController", request[:controller]
+      assert_equal "index", request[:action]      
+      assert_equal "GET", request[:method]
+      assert_equal '10.1.1.33', request[:ip]
+      assert_equal '2008-07-13 06:25:58', request[:timestamp]            
+    end
+
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
+      assert_equal "http://example.com/employee.xml", request[:url]
+      assert_equal 200, request[:status]
+      assert_equal 0.21665, request[:duration]
+      assert_equal 0.00926, request[:rendering]
+      assert_equal 0.0, request[:db]
+    end
+
+  end
+  
+end

data/test/record_inserter_test.rb

@@ -0,0 +1,45 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/base/record_inserter"
+
+require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
+
+class RecordInserterTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
+  end
+  
+  def setup
+    File.delete('_tmp.db') if File.exist?('_tmp.db')
+  end
+  
+  def teardown
+    File.delete('_tmp.db') if File.exist?('_tmp.db')
+  end
+  
+  def test_insert_log_fragment
+    
+    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
+    end
+
+    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
+    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i 
+  end
+  
+  def test_insert_multiple_fragments
+    RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
+    end
+
+    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(2)).each { |request| batch.insert(request) }
+    end
+    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
+    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i    
+  end
+  
+end