wvanbergen-request-log-analyzer 0.1.2 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +17 -4
- data/Rakefile +4 -3
- data/TODO +1 -3
- data/bin/request-log-analyzer +28 -8
- data/bin/request-log-database +2 -0
- data/lib/base/log_parser.rb +78 -0
- data/lib/base/record_inserter.rb +139 -0
- data/lib/base/summarizer.rb +71 -0
- data/lib/merb_analyzer/log_parser.rb +26 -0
- data/lib/merb_analyzer/summarizer.rb +61 -0
- data/lib/rails_analyzer/log_parser.rb +20 -68
- data/lib/rails_analyzer/record_inserter.rb +1 -123
- data/lib/rails_analyzer/summarizer.rb +16 -64
- data/output/hourly_spread.rb +1 -1
- data/output/timespan.rb +8 -1
- data/output/usage.rb +1 -0
- data/test/log_fragments/fragment_3.log +12 -0
- data/test/log_fragments/merb_1.log +84 -0
- data/test/merb_log_parser_test.rb +39 -0
- data/test/{log_parser_test.rb → rails_log_parser_test.rb} +15 -5
- data/test/record_inserter_test.rb +3 -0
- data/test/tasks.rake +1 -1
- metadata +15 -4
data/README
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
Request log analyzer
|
2
2
|
--------------------------------
|
3
3
|
|
4
|
-
This is a simple command line tool to analyze request log files
|
5
|
-
|
6
|
-
as well. Its purpose is to find what actions are best candidates for optimization.
|
4
|
+
This is a simple command line tool to analyze request log files of both Rails and
|
5
|
+
Merb. Its purpose is to find what actions are best candidates for optimization.
|
7
6
|
|
8
7
|
This tool will parse all requests in the logfile and aggregate the
|
9
8
|
information. Once it is finished parsing the log file, it will show the
|
@@ -46,6 +45,7 @@ Successfully analyzed 58908 requests from log file
|
|
46
45
|
Timestamp first request: 2008-07-13T06:25:58+00:00
|
47
46
|
Timestamp last request: 2008-07-20T06:18:53+00:00
|
48
47
|
Total time analyzed: 7 days
|
48
|
+
Methods: DELETE (1%), GET (50%), POST (22%), PUT (25%).
|
49
49
|
|
50
50
|
Top 10 most requested actions
|
51
51
|
========================================================================
|
@@ -92,7 +92,7 @@ Top 10 worst DB offenders - mean time
|
|
92
92
|
/overview/:date/ : 0.467s [19359 requests]
|
93
93
|
........
|
94
94
|
|
95
|
-
Mongrel process blockers (> 1.0 seconds)
|
95
|
+
Mongrel process blockers (> 1.0 seconds)
|
96
96
|
========================================================================
|
97
97
|
/overview/:date/ : 7494.233s [3144 requests]
|
98
98
|
/overview/ : 8320.293s [1549 requests]
|
@@ -117,3 +117,16 @@ Requests graph - per hour
|
|
117
117
|
17:00 - 1598 : XXXX
|
118
118
|
18:00 - 792 : XX
|
119
119
|
........
|
120
|
+
|
121
|
+
Errors
|
122
|
+
========================================================================
|
123
|
+
ArgumentError: [237 requests]
|
124
|
+
-> invalid date
|
125
|
+
StaleObjectError: [28 requests]
|
126
|
+
-> Attempted to update a stale object
|
127
|
+
RuntimeError: [3 requests]
|
128
|
+
-> Cannot destroy rule before it was created
|
129
|
+
StatementError: [2 requests]
|
130
|
+
-> Mysql::Error: Deadlock found when trying to get lock; try restarting transaction
|
131
|
+
NoMethodError: [1 requests]
|
132
|
+
-> undefined method `code' for nil:NilClass
|
data/Rakefile
CHANGED
@@ -2,14 +2,14 @@ require 'rubygems'
|
|
2
2
|
|
3
3
|
load 'test/tasks.rake'
|
4
4
|
|
5
|
-
desc 'Default: run unit tests.'
|
5
|
+
desc 'Default: run unit tests for request-log-analyzer.'
|
6
6
|
task :default => :test
|
7
7
|
|
8
8
|
|
9
9
|
namespace :gem do
|
10
10
|
|
11
|
-
desc "Sets the version and date of the gem"
|
12
|
-
task :version do
|
11
|
+
desc "Sets the version and date of the gem. Requires the VERSION environment variable."
|
12
|
+
task :version => [:manifest] do
|
13
13
|
|
14
14
|
require 'date'
|
15
15
|
|
@@ -24,6 +24,7 @@ namespace :gem do
|
|
24
24
|
File.open(spec_file, 'w') { |f| f << spec }
|
25
25
|
end
|
26
26
|
|
27
|
+
desc "Creates a git tag for the provided VERSION"
|
27
28
|
task :tag => [:version] do
|
28
29
|
|
29
30
|
new_version = ENV['VERSION']
|
data/TODO
CHANGED
@@ -2,9 +2,6 @@ TODO items for Rails-log-analyzer
|
|
2
2
|
=================================
|
3
3
|
Contact willem AT vanbergen DOT org if you want to help out with the development.
|
4
4
|
|
5
|
-
Summarizer:
|
6
|
-
- Look at request types (GET / POST)
|
7
|
-
|
8
5
|
Database:
|
9
6
|
- Add query functionality for the resulting database file (interactive reports?)
|
10
7
|
- Link request processing line to request completed line
|
@@ -17,4 +14,5 @@ Rails integration:
|
|
17
14
|
General:
|
18
15
|
- Add useful rake tasks
|
19
16
|
- Add more tests
|
17
|
+
- Fix multiple file handling
|
20
18
|
- World domination
|
data/bin/request-log-analyzer
CHANGED
@@ -1,14 +1,18 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
require File.dirname(__FILE__) + '/../lib/command_line/arguments'
|
3
|
+
require File.dirname(__FILE__) + '/../lib/base/log_parser'
|
4
|
+
require File.dirname(__FILE__) + '/../lib/base/summarizer'
|
3
5
|
require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
|
4
6
|
require File.dirname(__FILE__) + '/../lib/rails_analyzer/summarizer'
|
7
|
+
require File.dirname(__FILE__) + '/../lib/merb_analyzer/log_parser'
|
8
|
+
require File.dirname(__FILE__) + '/../lib/merb_analyzer/summarizer'
|
5
9
|
require File.dirname(__FILE__) + '/../lib/bashcolorizer'
|
6
10
|
require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
|
7
11
|
|
8
12
|
puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
|
9
13
|
|
10
14
|
# Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
|
11
|
-
# This is used to aggregate simular requests.
|
15
|
+
# This is used to aggregate simular requests.
|
12
16
|
# <tt>request</tt> The request to evaluate.
|
13
17
|
# Returns uniformed url string.
|
14
18
|
# Raises on mailformed request.
|
@@ -17,9 +21,9 @@ def request_hasher(request)
|
|
17
21
|
url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
|
18
22
|
url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
|
19
23
|
|
20
|
-
url.gsub!(/\/\d+-\d+-\d
|
21
|
-
url.gsub!(/\/\d+-\d
|
22
|
-
url.gsub!(/\/\d
|
24
|
+
url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
|
25
|
+
url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
|
26
|
+
url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
|
23
27
|
|
24
28
|
return url
|
25
29
|
elsif request[:controller] && request[:action]
|
@@ -48,6 +52,7 @@ begin
|
|
48
52
|
command_line.switch(:guess_database_time, :g)
|
49
53
|
command_line.switch(:fast, :f)
|
50
54
|
command_line.switch(:colorize, :z)
|
55
|
+
command_line.switch(:merb, :m)
|
51
56
|
command_line.flag(:output, :alias => :o)
|
52
57
|
command_line.flag(:amount, :alias => :c)
|
53
58
|
command_line.required_files = 1
|
@@ -60,15 +65,30 @@ rescue CommandLine::Error => e
|
|
60
65
|
exit(0)
|
61
66
|
end
|
62
67
|
|
63
|
-
$summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
|
64
|
-
$summarizer.blocker_duration = 1.0
|
65
68
|
|
66
|
-
|
69
|
+
if $arguments[:merb]
|
70
|
+
$summarizer = MerbAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
|
71
|
+
else
|
72
|
+
$summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
|
73
|
+
end
|
74
|
+
|
75
|
+
if $arguments[:fast]
|
76
|
+
line_types = [:completed]
|
77
|
+
elsif $arguments[:merb]
|
78
|
+
line_types = MerbAnalyzer::LogParser::LOG_LINES.keys
|
79
|
+
else
|
80
|
+
line_types = RailsAnalyzer::LogParser::LOG_LINES.keys
|
81
|
+
end
|
67
82
|
|
68
83
|
# Walk through al the files given via the arguments.
|
69
84
|
$arguments.files.each do |log_file|
|
70
85
|
puts "Processing #{line_types.join(', ')} log lines from #{log_file}..."
|
71
|
-
|
86
|
+
|
87
|
+
if $arguments[:merb]
|
88
|
+
parser = MerbAnalyzer::LogParser.new(log_file)
|
89
|
+
else
|
90
|
+
parser = RailsAnalyzer::LogParser.new(log_file)
|
91
|
+
end
|
72
92
|
|
73
93
|
# add progress bar
|
74
94
|
unless $arguments[:fast]
|
data/bin/request-log-database
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
#!/usr/bin/ruby
|
2
2
|
|
3
3
|
require File.dirname(__FILE__) + '/../lib/command_line/arguments'
|
4
|
+
require File.dirname(__FILE__) + '/../lib/base/log_parser'
|
5
|
+
require File.dirname(__FILE__) + '/../lib/base/record_inserter'
|
4
6
|
require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
|
5
7
|
require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
|
6
8
|
require File.dirname(__FILE__) + '/../lib/bashcolorizer'
|
@@ -0,0 +1,78 @@
|
|
1
|
+
module Base
|
2
|
+
# Parse a log file
|
3
|
+
class LogParser
|
4
|
+
|
5
|
+
LOG_LINES = {}
|
6
|
+
|
7
|
+
# LogParser initializer
|
8
|
+
# <tt>file</tt> The fileobject this LogParser wil operate on.
|
9
|
+
def initialize(file, options = {})
|
10
|
+
@file_name = file
|
11
|
+
@options = options
|
12
|
+
@file_size = File.size(@file_name)
|
13
|
+
|
14
|
+
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
15
|
+
end
|
16
|
+
|
17
|
+
def progress(&block)
|
18
|
+
@progress_handler = block
|
19
|
+
end
|
20
|
+
|
21
|
+
# Output a warning
|
22
|
+
# <tt>message</tt> The warning message (object)
|
23
|
+
def warn(message)
|
24
|
+
puts " -> " + message.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
def convert_value(value, type)
|
28
|
+
return case type
|
29
|
+
when :string; value.to_s
|
30
|
+
when :int; value.to_i
|
31
|
+
when :sec; value.to_f
|
32
|
+
when :msec; value.to_f / 1000
|
33
|
+
when :timestamp; value.to_s # TODO: fix me?
|
34
|
+
else
|
35
|
+
warn("Unkwn type encountered: #{type}")
|
36
|
+
value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Finds a log line and then parses the information in the line.
|
41
|
+
# Yields a hash containing the information found.
|
42
|
+
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
43
|
+
# Yeilds a Hash when it encounters a chunk of information.
|
44
|
+
def each(*line_types, &block)
|
45
|
+
log_lines_hash = self.class::LOG_LINES
|
46
|
+
|
47
|
+
|
48
|
+
# parse everything by default
|
49
|
+
line_types = log_lines_hash.keys if line_types.empty?
|
50
|
+
|
51
|
+
File.open(@file_name) do |file|
|
52
|
+
|
53
|
+
file.each_line do |line|
|
54
|
+
|
55
|
+
@progress_handler.call(file.pos, @file_size) if @progress_handler
|
56
|
+
|
57
|
+
line_types.each do |line_type|
|
58
|
+
if log_lines_hash[line_type][:teaser] =~ line
|
59
|
+
if log_lines_hash[line_type][:regexp] =~ line
|
60
|
+
|
61
|
+
captures = $~.captures
|
62
|
+
request = { :type => line_type, :line => file.lineno }
|
63
|
+
log_lines_hash[line_type][:params].each_with_index do |paramhash, index|
|
64
|
+
paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
|
65
|
+
end
|
66
|
+
|
67
|
+
yield(request) if block_given?
|
68
|
+
else
|
69
|
+
warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
@progress_handler.call(:finished, @file_size) if @progress_handler
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
module Base
|
5
|
+
|
6
|
+
# Set of functions that can be used to easily log requests into a SQLite3 Database.
|
7
|
+
class RecordInserter
|
8
|
+
|
9
|
+
attr_reader :database
|
10
|
+
attr_reader :current_request
|
11
|
+
attr_reader :warning_count
|
12
|
+
|
13
|
+
# Initializer
|
14
|
+
# <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
|
15
|
+
def initialize(db_file, options = {})
|
16
|
+
@database = SQLite3::Database.new(db_file)
|
17
|
+
@insert_statements = nil
|
18
|
+
@warning_count = 0
|
19
|
+
create_tables_if_needed!
|
20
|
+
|
21
|
+
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Calculate the database durations of the requests currenty in the database.
|
25
|
+
# Used if a logfile does contain any database durations.
|
26
|
+
def calculate_db_durations!
|
27
|
+
@database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
|
28
|
+
end
|
29
|
+
|
30
|
+
# Insert a batch of loglines into the database.
|
31
|
+
# Function prepares insert statements, yeilds and then closes and commits.
|
32
|
+
def insert_batch(&block)
|
33
|
+
@database.transaction
|
34
|
+
prepare_statements!
|
35
|
+
block.call(self)
|
36
|
+
close_prepared_statements!
|
37
|
+
@database.commit
|
38
|
+
rescue Exception => e
|
39
|
+
puts e.message
|
40
|
+
@database.rollback
|
41
|
+
end
|
42
|
+
|
43
|
+
def insert_warning(line, warning)
|
44
|
+
@database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
|
45
|
+
@warning_count += 1
|
46
|
+
end
|
47
|
+
|
48
|
+
# Insert a request into the database.
|
49
|
+
# def insert(request, close_statements = false)
|
50
|
+
# raise 'No insert defined for this log file type'
|
51
|
+
# end
|
52
|
+
|
53
|
+
# Insert a batch of files into the database.
|
54
|
+
# <tt>db_file</tt> The filename of the database file to use.
|
55
|
+
# Returns the created database.
|
56
|
+
def self.insert_batch_into(db_file, options = {}, &block)
|
57
|
+
db = self.new(db_file)
|
58
|
+
db.insert_batch(&block)
|
59
|
+
return db
|
60
|
+
end
|
61
|
+
|
62
|
+
def count(type)
|
63
|
+
@database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
|
64
|
+
end
|
65
|
+
|
66
|
+
protected
|
67
|
+
|
68
|
+
# Prepare insert statements.
|
69
|
+
def prepare_statements!
|
70
|
+
@insert_statements = {
|
71
|
+
:started => @database.prepare("
|
72
|
+
INSERT INTO started_requests ( line, timestamp, ip, method, controller, action)
|
73
|
+
VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
|
74
|
+
|
75
|
+
:failed => @database.prepare("
|
76
|
+
INSERT INTO failed_requests ( line, exception_string, stack_trace, error)
|
77
|
+
VALUES (:line, :exception_string, :stack_trace, :error)"),
|
78
|
+
|
79
|
+
:completed => @database.prepare("
|
80
|
+
INSERT INTO completed_requests ( line, url, status, duration, rendering_time, database_time)
|
81
|
+
VALUES (:line, :url, :status, :duration, :rendering, :db)")
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
# Close all prepared statments
|
86
|
+
def close_prepared_statements!
|
87
|
+
@insert_statements.each { |key, stmt| stmt.close }
|
88
|
+
end
|
89
|
+
|
90
|
+
# Create the needed database tables if they don't exist.
|
91
|
+
def create_tables_if_needed!
|
92
|
+
|
93
|
+
@database.execute("
|
94
|
+
CREATE TABLE IF NOT EXISTS started_requests (
|
95
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
96
|
+
line INTEGER NOT NULL,
|
97
|
+
timestamp DATETIME NOT NULL,
|
98
|
+
controller VARCHAR(255) NOT NULL,
|
99
|
+
action VARCHAR(255) NOT NULL,
|
100
|
+
method VARCHAR(6) NOT NULL,
|
101
|
+
ip VARCHAR(6) NOT NULL
|
102
|
+
)
|
103
|
+
");
|
104
|
+
|
105
|
+
@database.execute("
|
106
|
+
CREATE TABLE IF NOT EXISTS failed_requests (
|
107
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
108
|
+
line INTEGER NOT NULL,
|
109
|
+
started_request_id INTEGER,
|
110
|
+
error VARCHAR(255),
|
111
|
+
exception_string VARCHAR(255),
|
112
|
+
stack_trace TEXT
|
113
|
+
)
|
114
|
+
");
|
115
|
+
|
116
|
+
@database.execute("
|
117
|
+
CREATE TABLE IF NOT EXISTS completed_requests (
|
118
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
119
|
+
line INTEGER NOT NULL,
|
120
|
+
started_request_id INTEGER,
|
121
|
+
url VARCHAR(255) NOT NULL,
|
122
|
+
hashed_url VARCHAR(255),
|
123
|
+
status INTEGER NOT NULL,
|
124
|
+
duration FLOAT,
|
125
|
+
rendering_time FLOAT,
|
126
|
+
database_time FLOAT
|
127
|
+
)
|
128
|
+
");
|
129
|
+
|
130
|
+
@database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
|
131
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
132
|
+
line INTEGER NOT NULL,
|
133
|
+
warning VARCHAR(255) NOT NULL
|
134
|
+
)
|
135
|
+
");
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module Base
|
2
|
+
|
3
|
+
# Functions to summarize an array of requets.
|
4
|
+
# Can calculate request counts, duratations, mean times etc. of all the requests given.
|
5
|
+
class Summarizer
|
6
|
+
attr_reader :actions
|
7
|
+
attr_reader :errors
|
8
|
+
attr_reader :request_count
|
9
|
+
attr_reader :request_time_graph
|
10
|
+
attr_reader :first_request_at
|
11
|
+
attr_reader :last_request_at
|
12
|
+
attr_reader :methods
|
13
|
+
|
14
|
+
attr_accessor :blocker_duration
|
15
|
+
DEFAULT_BLOCKER_DURATION = 1.0
|
16
|
+
|
17
|
+
# Initializer. Sets global variables
|
18
|
+
# Options
|
19
|
+
def initialize(options = {})
|
20
|
+
@actions = {}
|
21
|
+
@blockers = {}
|
22
|
+
@errors = {}
|
23
|
+
@request_count = 0
|
24
|
+
@blocker_duration = DEFAULT_BLOCKER_DURATION
|
25
|
+
@request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
26
|
+
@methods = {:GET => 0, :POST => 0, :PUT => 0, :DELETE => 0, :unknown => 0}
|
27
|
+
|
28
|
+
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Check if any of the request parsed had a timestamp.
|
32
|
+
def has_timestamps?
|
33
|
+
@first_request_at
|
34
|
+
end
|
35
|
+
|
36
|
+
# Calculate the duration of a request
|
37
|
+
# Returns a DateTime object if possible, 0 otherwise.
|
38
|
+
def duration
|
39
|
+
(@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).ceil : 0
|
40
|
+
end
|
41
|
+
|
42
|
+
# Check if the request time graph usable data.
|
43
|
+
def request_time_graph?
|
44
|
+
@request_time_graph.uniq != [0] && duration > 0
|
45
|
+
end
|
46
|
+
|
47
|
+
# Return a list of requests sorted on a specific action field
|
48
|
+
# <tt>field</tt> The action field to sort by.
|
49
|
+
# <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
|
50
|
+
def sort_actions_by(field, min_count = nil)
|
51
|
+
actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
|
52
|
+
actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
|
53
|
+
end
|
54
|
+
|
55
|
+
# Returns a list of request blockers sorted by a specific field
|
56
|
+
# <tt>field</tt> The action field to sort by.
|
57
|
+
# <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
|
58
|
+
def sort_blockers_by(field, min_count = @blocker_duration)
|
59
|
+
blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
|
60
|
+
blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] }
|
61
|
+
end
|
62
|
+
|
63
|
+
# Returns a list of request blockers sorted by a specific field
|
64
|
+
# <tt>field</tt> The action field to sort by.
|
65
|
+
# <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
|
66
|
+
def sort_errors_by(field, min_count = nil)
|
67
|
+
errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
|
68
|
+
errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module MerbAnalyzer
|
2
|
+
|
3
|
+
class LogParser < Base::LogParser
|
4
|
+
LOG_LINES = {
|
5
|
+
# ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
6
|
+
:started => {
|
7
|
+
:teaser => /Started/,
|
8
|
+
:regexp => /Started request handling\:\ (.+)/,
|
9
|
+
:params => [{:timestamp => :timestamp}]
|
10
|
+
},
|
11
|
+
# ~ Params: {"action"=>"create", "controller"=>"session"}
|
12
|
+
# ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
|
13
|
+
:params => {
|
14
|
+
:teaser => /Params/,
|
15
|
+
:regexp => /Params\:\ \{(.+)\}/,
|
16
|
+
:params => [{:raw_hash => :string}]
|
17
|
+
},
|
18
|
+
# ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
|
19
|
+
:completed => {
|
20
|
+
:teaser => /\{:dispatch_time/,
|
21
|
+
:regexp => /\{\:dispatch_time=>(.+), (?:\:after_filters_time=>(.+), )?(?:\:before_filters_time=>(.+), )?\:action_time=>(.+)\}/,
|
22
|
+
:params => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
|
23
|
+
}
|
24
|
+
}
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'date'
|
2
|
+
module MerbAnalyzer
|
3
|
+
|
4
|
+
# Functions to summarize an array of requets.
|
5
|
+
# Can calculate request counts, duratations, mean times etc. of all the requests given.
|
6
|
+
class Summarizer < Base::Summarizer
|
7
|
+
|
8
|
+
# Initializer. Sets global variables
|
9
|
+
def initialize_hook(options = {})
|
10
|
+
@hash_cache = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
# Parse a request string into a hash containing all keys found in the string.
|
14
|
+
# Yields the hash found to the block operator.
|
15
|
+
# <tt>request</tt> The request string to parse.
|
16
|
+
# <tt>&block</tt> Block operator
|
17
|
+
def group(request, &block)
|
18
|
+
request[:duration] ||= 0
|
19
|
+
|
20
|
+
case request[:type]
|
21
|
+
when :started
|
22
|
+
@first_request_at ||= request[:timestamp] # assume time-based order of file
|
23
|
+
@last_request_at = request[:timestamp] # assume time-based order of file
|
24
|
+
@request_time_graph[request[:timestamp][11..12].to_i] +=1
|
25
|
+
|
26
|
+
when :params
|
27
|
+
params_hash = {}
|
28
|
+
request[:raw_hash].split(',').collect{|x| x.split('=>')}.each do |k,v|
|
29
|
+
key = k.gsub('"', '').gsub(' ', '').to_sym
|
30
|
+
value = v.gsub('"', '')
|
31
|
+
request.store(key, value)
|
32
|
+
end
|
33
|
+
|
34
|
+
hash = block_given? ? yield(request) : request.hash
|
35
|
+
|
36
|
+
@actions[hash] ||= {:count => 0, :total_time => 0.0, :total_db_time => 0.0, :total_rendering_time => 0.0,
|
37
|
+
:min_time => request[:duration], :max_time => request[:duration] }
|
38
|
+
|
39
|
+
@actions[hash][:count] += 1
|
40
|
+
request[:method] = 'GET' unless request[:method]
|
41
|
+
@methods[request[:method].upcase.to_sym] += 1
|
42
|
+
|
43
|
+
@hash_cache = hash
|
44
|
+
when :completed
|
45
|
+
@request_count += 1
|
46
|
+
|
47
|
+
@actions[@hash_cache][:total_time] += request[:dispatch_time]
|
48
|
+
@actions[@hash_cache][:mean_time] = @actions[@hash_cache][:total_time] / @actions[@hash_cache][:count].to_f
|
49
|
+
@actions[@hash_cache][:min_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].min
|
50
|
+
@actions[@hash_cache][:max_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].max
|
51
|
+
|
52
|
+
@actions[@hash_cache][:total_db_time] = 0
|
53
|
+
@actions[@hash_cache][:mean_db_time] = 0
|
54
|
+
@actions[@hash_cache][:mean_rendering_time] = 0
|
55
|
+
|
56
|
+
when :failed
|
57
|
+
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -1,83 +1,35 @@
|
|
1
|
-
require 'date'
|
2
|
-
|
3
1
|
module RailsAnalyzer
|
4
|
-
# Parse a rails log file
|
5
|
-
class LogParser
|
6
2
|
|
3
|
+
class LogParser < Base::LogParser
|
4
|
+
|
5
|
+
# Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
|
6
|
+
RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
|
7
|
+
|
8
|
+
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
9
|
+
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
|
10
|
+
|
11
|
+
|
7
12
|
LOG_LINES = {
|
8
13
|
# Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
|
9
14
|
:started => {
|
10
15
|
:teaser => /Processing/,
|
11
|
-
:regexp => /Processing (
|
12
|
-
:params => {
|
16
|
+
:regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
|
17
|
+
:params => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
|
13
18
|
},
|
14
19
|
# RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy'
|
15
20
|
:failed => {
|
16
21
|
:teaser => /Error/,
|
17
|
-
:regexp => /(\w+)(Error|Invalid) \((.*)\)\:(.*)/,
|
18
|
-
:params => {
|
22
|
+
:regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
|
23
|
+
:params => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
|
19
24
|
},
|
20
|
-
|
25
|
+
|
21
26
|
:completed => {
|
22
|
-
:teaser
|
23
|
-
:regexp
|
24
|
-
:params
|
27
|
+
:teaser => /Completed in /,
|
28
|
+
:regexp => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
|
29
|
+
:params => [{:duration => :sec}, {:rendering => :sec}, {:db => :sec}, {:status => :int}, {:url => :string}, # 2.1 variant
|
30
|
+
{:duration => :msec}, {:rendering => :msec}, {:db => :msec}, {:status => :int}, {:url => :string}] # 2.2 variant
|
31
|
+
|
25
32
|
}
|
26
33
|
}
|
27
|
-
|
28
|
-
# LogParser initializer
|
29
|
-
# <tt>file</tt> The fileobject this LogParser wil operate on.
|
30
|
-
def initialize(file, options = {})
|
31
|
-
@file_name = file
|
32
|
-
@options = options
|
33
|
-
@file_size = File.size(@file_name)
|
34
|
-
end
|
35
|
-
|
36
|
-
def progress(&block)
|
37
|
-
@progress_handler = block
|
38
|
-
end
|
39
|
-
|
40
|
-
# Output a warning
|
41
|
-
# <tt>message</tt> The warning message (object)
|
42
|
-
def warn(message)
|
43
|
-
puts " -> " + message.to_s
|
44
|
-
end
|
45
|
-
|
46
|
-
# Finds a log line and then parses the information in the line.
|
47
|
-
# Yields a hash containing the information found.
|
48
|
-
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
49
|
-
# Yeilds a Hash when it encounters a chunk of information.
|
50
|
-
def each(*line_types, &block)
|
51
|
-
# parse everything by default
|
52
|
-
line_types = LOG_LINES.keys if line_types.empty?
|
53
|
-
|
54
|
-
File.open(@file_name) do |file|
|
55
|
-
|
56
|
-
file.each_line do |line|
|
57
|
-
|
58
|
-
@progress_handler.call(file.pos, @file_size) if @progress_handler
|
59
|
-
|
60
|
-
line_types.each do |line_type|
|
61
|
-
if LOG_LINES[line_type][:teaser] =~ line
|
62
|
-
if LOG_LINES[line_type][:regexp] =~ line
|
63
|
-
request = { :type => line_type, :line => file.lineno }
|
64
|
-
LOG_LINES[line_type][:params].each do |key, value|
|
65
|
-
request[key] = case value
|
66
|
-
when Numeric; $~[value]
|
67
|
-
when Array; $~[value.first].send(value.last)
|
68
|
-
else; nil
|
69
|
-
end
|
70
|
-
|
71
|
-
end
|
72
|
-
yield(request) if block_given?
|
73
|
-
else
|
74
|
-
warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
@progress_handler.call(:finished, @file_size) if @progress_handler
|
80
|
-
end
|
81
|
-
end
|
82
34
|
end
|
83
|
-
end
|
35
|
+
end
|
@@ -4,45 +4,8 @@ require 'sqlite3'
|
|
4
4
|
module RailsAnalyzer
|
5
5
|
|
6
6
|
# Set of functions that can be used to easily log requests into a SQLite3 Database.
|
7
|
-
class RecordInserter
|
7
|
+
class RecordInserter < Base::RecordInserter
|
8
8
|
|
9
|
-
attr_reader :database
|
10
|
-
attr_reader :current_request
|
11
|
-
attr_reader :warning_count
|
12
|
-
|
13
|
-
# Initializer
|
14
|
-
# <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
|
15
|
-
def initialize(db_file)
|
16
|
-
@database = SQLite3::Database.new(db_file)
|
17
|
-
@insert_statements = nil
|
18
|
-
@warning_count = 0
|
19
|
-
create_tables_if_needed!
|
20
|
-
end
|
21
|
-
|
22
|
-
# Calculate the database durations of the requests currenty in the database.
|
23
|
-
# Used if a logfile does contain any database durations.
|
24
|
-
def calculate_db_durations!
|
25
|
-
@database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
|
26
|
-
end
|
27
|
-
|
28
|
-
# Insert a batch of loglines into the database.
|
29
|
-
# Function prepares insert statements, yeilds and then closes and commits.
|
30
|
-
def insert_batch(&block)
|
31
|
-
@database.transaction
|
32
|
-
prepare_statements!
|
33
|
-
block.call(self)
|
34
|
-
close_prepared_statements!
|
35
|
-
@database.commit
|
36
|
-
rescue Exception => e
|
37
|
-
puts e.message
|
38
|
-
@database.rollback
|
39
|
-
end
|
40
|
-
|
41
|
-
def insert_warning(line, warning)
|
42
|
-
@database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
|
43
|
-
@warning_count += 1
|
44
|
-
end
|
45
|
-
|
46
9
|
# Insert a request into the database.
|
47
10
|
# <tt>request</tt> The request to insert.
|
48
11
|
# <tt>close_statements</tt> Close prepared statements (default false)
|
@@ -72,90 +35,5 @@ module RailsAnalyzer
|
|
72
35
|
close_prepared_statements! if close_statements
|
73
36
|
end
|
74
37
|
|
75
|
-
# Insert a batch of files into the database.
|
76
|
-
# <tt>db_file</tt> The filename of the database file to use.
|
77
|
-
# Returns the created database.
|
78
|
-
def self.insert_batch_into(db_file, options = {}, &block)
|
79
|
-
db = RecordInserter.new(db_file)
|
80
|
-
db.insert_batch(&block)
|
81
|
-
return db
|
82
|
-
end
|
83
|
-
|
84
|
-
def count(type)
|
85
|
-
@database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
|
86
|
-
end
|
87
|
-
|
88
|
-
protected
|
89
|
-
|
90
|
-
# Prepare insert statements.
|
91
|
-
def prepare_statements!
|
92
|
-
@insert_statements = {
|
93
|
-
:started => @database.prepare("
|
94
|
-
INSERT INTO started_requests ( line, timestamp, ip, method, controller, action)
|
95
|
-
VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
|
96
|
-
|
97
|
-
:failed => @database.prepare("
|
98
|
-
INSERT INTO failed_requests ( line, exception_string, stack_trace, error)
|
99
|
-
VALUES (:line, :exception_string, :stack_trace, :error)"),
|
100
|
-
|
101
|
-
:completed => @database.prepare("
|
102
|
-
INSERT INTO completed_requests ( line, url, status, duration, rendering_time, database_time)
|
103
|
-
VALUES (:line, :url, :status, :duration, :rendering, :db)")
|
104
|
-
}
|
105
|
-
end
|
106
|
-
|
107
|
-
# Close all prepared statments
|
108
|
-
def close_prepared_statements!
|
109
|
-
@insert_statements.each { |key, stmt| stmt.close }
|
110
|
-
end
|
111
|
-
|
112
|
-
# Create the needed database tables if they don't exist.
|
113
|
-
def create_tables_if_needed!
|
114
|
-
|
115
|
-
@database.execute("
|
116
|
-
CREATE TABLE IF NOT EXISTS started_requests (
|
117
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
118
|
-
line INTEGER NOT NULL,
|
119
|
-
timestamp DATETIME NOT NULL,
|
120
|
-
controller VARCHAR(255) NOT NULL,
|
121
|
-
action VARCHAR(255) NOT NULL,
|
122
|
-
method VARCHAR(6) NOT NULL,
|
123
|
-
ip VARCHAR(6) NOT NULL
|
124
|
-
)
|
125
|
-
");
|
126
|
-
|
127
|
-
@database.execute("
|
128
|
-
CREATE TABLE IF NOT EXISTS failed_requests (
|
129
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
130
|
-
line INTEGER NOT NULL,
|
131
|
-
started_request_id INTEGER,
|
132
|
-
error VARCHAR(255),
|
133
|
-
exception_string VARCHAR(255),
|
134
|
-
stack_trace TEXT
|
135
|
-
)
|
136
|
-
");
|
137
|
-
|
138
|
-
@database.execute("
|
139
|
-
CREATE TABLE IF NOT EXISTS completed_requests (
|
140
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
141
|
-
line INTEGER NOT NULL,
|
142
|
-
started_request_id INTEGER,
|
143
|
-
url VARCHAR(255) NOT NULL,
|
144
|
-
hashed_url VARCHAR(255),
|
145
|
-
status INTEGER NOT NULL,
|
146
|
-
duration FLOAT,
|
147
|
-
rendering_time FLOAT,
|
148
|
-
database_time FLOAT
|
149
|
-
)
|
150
|
-
");
|
151
|
-
|
152
|
-
@database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
|
153
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
154
|
-
line INTEGER NOT NULL,
|
155
|
-
warning VARCHAR(255) NOT NULL
|
156
|
-
)
|
157
|
-
");
|
158
|
-
end
|
159
|
-
|
160
38
|
end
|
161
39
|
end
|
@@ -1,37 +1,17 @@
|
|
1
|
+
require 'date'
|
1
2
|
module RailsAnalyzer
|
2
3
|
|
3
4
|
# Functions to summarize an array of requets.
|
4
5
|
# Can calculate request counts, duratations, mean times etc. of all the requests given.
|
5
|
-
class Summarizer
|
6
|
+
class Summarizer < Base::Summarizer
|
6
7
|
|
7
|
-
attr_reader :actions
|
8
|
-
attr_reader :errors
|
9
|
-
attr_reader :request_count
|
10
|
-
attr_reader :request_time_graph
|
11
|
-
attr_reader :first_request_at
|
12
|
-
attr_reader :last_request_at
|
13
|
-
|
14
|
-
attr_accessor :blocker_duration
|
15
|
-
DEFAULT_BLOCKER_DURATION = 1.0
|
16
|
-
|
17
8
|
# Initializer. Sets global variables
|
18
9
|
# Options
|
19
10
|
# * <tt>:calculate_database</tt> Calculate the database times if they are not explicitly logged.
|
20
|
-
def
|
21
|
-
@actions = {}
|
22
|
-
@blockers = {}
|
23
|
-
@errors = {}
|
24
|
-
@request_count = 0
|
25
|
-
@blocker_duration = DEFAULT_BLOCKER_DURATION
|
11
|
+
def initialize_hook(options = {})
|
26
12
|
@calculate_database = options[:calculate_database]
|
27
|
-
@request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
28
|
-
end
|
29
|
-
|
30
|
-
# Check if any of the request parsed had a timestamp.
|
31
|
-
def has_timestamps?
|
32
|
-
@first_request_at
|
33
13
|
end
|
34
|
-
|
14
|
+
|
35
15
|
# Parse a request string into a hash containing all keys found in the string.
|
36
16
|
# Yields the hash found to the block operator.
|
37
17
|
# <tt>request</tt> The request string to parse.
|
@@ -40,11 +20,18 @@ module RailsAnalyzer
|
|
40
20
|
request[:duration] ||= 0
|
41
21
|
|
42
22
|
case request[:type]
|
43
|
-
when :started
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
23
|
+
when :started
|
24
|
+
if request[:timestamp]
|
25
|
+
@first_request_at ||= request[:timestamp] # assume time-based order of file
|
26
|
+
@last_request_at = request[:timestamp] # assume time-based order of file
|
27
|
+
@request_time_graph[request[:timestamp][11..12].to_i] +=1
|
28
|
+
end
|
29
|
+
if request[:method]
|
30
|
+
@methods[request[:method].to_sym] ||= 0
|
31
|
+
@methods[request[:method].to_sym] += 1
|
32
|
+
else
|
33
|
+
@methods[:unknown] += 1
|
34
|
+
end
|
48
35
|
when :completed
|
49
36
|
@request_count += 1
|
50
37
|
hash = block_given? ? yield(request) : request.hash
|
@@ -82,40 +69,5 @@ module RailsAnalyzer
|
|
82
69
|
end
|
83
70
|
end
|
84
71
|
|
85
|
-
# Return a list of requests sorted on a specific action field
|
86
|
-
# <tt>field</tt> The action field to sort by.
|
87
|
-
# <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
|
88
|
-
def sort_actions_by(field, min_count = nil)
|
89
|
-
actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
|
90
|
-
actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
|
91
|
-
end
|
92
|
-
|
93
|
-
# Returns a list of request blockers sorted by a specific field
|
94
|
-
# <tt>field</tt> The action field to sort by.
|
95
|
-
# <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
|
96
|
-
def sort_blockers_by(field, min_count = @blocker_duration)
|
97
|
-
blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
|
98
|
-
blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] }
|
99
|
-
end
|
100
|
-
|
101
|
-
# Returns a list of request blockers sorted by a specific field
|
102
|
-
# <tt>field</tt> The action field to sort by.
|
103
|
-
# <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
|
104
|
-
def sort_errors_by(field, min_count = nil)
|
105
|
-
errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
|
106
|
-
errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] }
|
107
|
-
end
|
108
|
-
|
109
|
-
# Calculate the duration of a request
|
110
|
-
# Returns a DateTime object if possible, 0 otherwise.
|
111
|
-
def duration
|
112
|
-
(@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).round : 0
|
113
|
-
end
|
114
|
-
|
115
|
-
# Check if the request time graph usable data.
|
116
|
-
def request_time_graph?
|
117
|
-
@request_time_graph.uniq != [0] && duration > 0
|
118
|
-
end
|
119
|
-
|
120
72
|
end
|
121
73
|
end
|
data/output/hourly_spread.rb
CHANGED
@@ -7,7 +7,7 @@ if $summarizer.request_time_graph?
|
|
7
7
|
color_cutoff = 15
|
8
8
|
|
9
9
|
puts
|
10
|
-
puts "Requests graph - per hour"
|
10
|
+
puts "Requests graph - average per day per hour"
|
11
11
|
puts green("========================================================================")
|
12
12
|
|
13
13
|
(0..23).each do |a|
|
data/output/timespan.rb
CHANGED
@@ -6,4 +6,11 @@ if $summarizer.has_timestamps?
|
|
6
6
|
puts "Timestamp first request: #{$summarizer.first_request_at}"
|
7
7
|
puts "Timestamp last request: #{$summarizer.last_request_at}"
|
8
8
|
puts "Total time analyzed: #{$summarizer.duration} days"
|
9
|
-
end
|
9
|
+
end
|
10
|
+
|
11
|
+
methods_print_array = []
|
12
|
+
methods_request_count = $summarizer.methods.inject(0) { |subtotal, (k, v)| subtotal + v }
|
13
|
+
$summarizer.methods.each do |key, value|
|
14
|
+
methods_print_array << "%s (%0.01f%%)" % [key, green((value * 100) / methods_request_count.to_f)]
|
15
|
+
end
|
16
|
+
puts 'Methods: ' + methods_print_array.join(', ') + '.'
|
data/output/usage.rb
CHANGED
@@ -7,6 +7,7 @@ puts " --guess-database-time, -g: Guesses the database duration of requests"
|
|
7
7
|
puts " --output, -o: Comma-separated list of reports to show"
|
8
8
|
puts " --amount, -c: Displays the top <amount> elements in the reports"
|
9
9
|
puts " --colorize, -z: Fancy bash coloring"
|
10
|
+
puts " --merb, -m: Parse merb logfiles"
|
10
11
|
puts
|
11
12
|
puts "Examples:"
|
12
13
|
puts " request-log-analyzer development.log"
|
@@ -0,0 +1,12 @@
|
|
1
|
+
Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
|
2
|
+
Parameters: {"action"=>"demo", "controller"=>"page"}
|
3
|
+
Logging in from session data...
|
4
|
+
Logged in as willem@depillem.com
|
5
|
+
Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain:
|
6
|
+
Rendering template within layouts/demo
|
7
|
+
Rendering page/demo
|
8
|
+
Rendered shared/_analytics (0.2ms)
|
9
|
+
Rendered layouts/_actions (0.6ms)
|
10
|
+
Rendered layouts/_menu (2.2ms)
|
11
|
+
Rendered layouts/_tabbar (0.5ms)
|
12
|
+
Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
@@ -0,0 +1,84 @@
|
|
1
|
+
~ Loaded DEVELOPMENT Environment...
|
2
|
+
~ Connecting to database...
|
3
|
+
~ loading gem 'merb_datamapper' ...
|
4
|
+
~ loading gem 'gettext' ...
|
5
|
+
~ loading gem 'merb_helpers' ...
|
6
|
+
~ loading gem 'merb-assets' ...
|
7
|
+
~ loading gem 'merb-action-args' ...
|
8
|
+
~ loading gem 'merb-mailer' ...
|
9
|
+
~ loading gem 'merb_param_protection' ...
|
10
|
+
~ loading gem 'merb_has_flash' ...
|
11
|
+
~ loading gem 'merb_forgery_protection' ...
|
12
|
+
~ loading gem 'dm-validations' ...
|
13
|
+
~ loading gem 'dm-timestamps' ...
|
14
|
+
~ loading gem 'dm-migrations' ...
|
15
|
+
~ loading gem 'dm-aggregates' ...
|
16
|
+
~ loading gem 'dm-adjust' ...
|
17
|
+
~ loading gem 'dm-serializer' ...
|
18
|
+
~ loading gem 'dm-constraints' ...
|
19
|
+
~ loading gem 'dm-timeline' ...
|
20
|
+
~ loading gem 'dm-searchable' ...
|
21
|
+
~ loading gem 'dm-audited' ...
|
22
|
+
~ loading gem 'lib/extensions' ...
|
23
|
+
~ loading gem 'lib/authenticated_system/authenticated_dependencies' ...
|
24
|
+
~ Compiling routes...
|
25
|
+
~ Using 'share-nothing' cookie sessions (4kb limit per client)
|
26
|
+
~ Using Mongrel adapter
|
27
|
+
~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
28
|
+
~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy", "method"=>"delete", "controller"=>"session"}
|
29
|
+
~ Cookie deleted: auth_token => nil
|
30
|
+
~ Redirecting to: / (302)
|
31
|
+
~ {:dispatch_time=>0.243424, :after_filters_time=>6.9e-05, :before_filters_time=>0.213213, :action_time=>0.241652}
|
32
|
+
~
|
33
|
+
|
34
|
+
~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
35
|
+
~ Params: {"action"=>"index", "controller"=>"dashboard"}
|
36
|
+
~ Redirecting to: /login (302)
|
37
|
+
~ {:dispatch_time=>0.002649, :after_filters_time=>7.4e-05, :action_time=>0.001951}
|
38
|
+
~
|
39
|
+
|
40
|
+
~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
41
|
+
~ Params: {"action"=>"create", "controller"=>"session"}
|
42
|
+
~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
|
43
|
+
~
|
44
|
+
|
45
|
+
~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
|
46
|
+
~ Params: {"authenticity_token"=>"[FILTERED]", "action"=>"create", "controller"=>"session", "login"=>"username", "password"=>"[FILTERED]", "remember_me"=>"0"}
|
47
|
+
~ Redirecting to: / (302)
|
48
|
+
~ {:dispatch_time=>0.006652, :after_filters_time=>0.000143, :before_filters_time=>0.000861, :action_time=>0.006171}
|
49
|
+
~
|
50
|
+
|
51
|
+
~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
|
52
|
+
~ Params: {"action"=>"index", "controller"=>"dashboard"}
|
53
|
+
~ Redirecting to: /dashboard (302)
|
54
|
+
~ {:dispatch_time=>0.008241, :after_filters_time=>0.000126, :before_filters_time=>0.002632, :action_time=>0.007711}
|
55
|
+
~
|
56
|
+
|
57
|
+
~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
|
58
|
+
~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
|
59
|
+
~ {:dispatch_time=>0.009458, :after_filters_time=>0.000103, :before_filters_time=>0.00266, :action_time=>0.008742}
|
60
|
+
~
|
61
|
+
|
62
|
+
~ Started request handling: Fri Aug 29 11:10:29 +0200 2008
|
63
|
+
~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
|
64
|
+
~ {:dispatch_time=>0.102725, :after_filters_time=>0.000115, :before_filters_time=>0.00411, :action_time=>0.101836}
|
65
|
+
~
|
66
|
+
|
67
|
+
~ Started request handling: Fri Aug 29 11:10:30 +0200 2008
|
68
|
+
~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"organisations"}
|
69
|
+
~ {:dispatch_time=>0.042575, :after_filters_time=>8.9e-05, :before_filters_time=>0.004267, :action_time=>0.041762}
|
70
|
+
~
|
71
|
+
|
72
|
+
~ Started request handling: Fri Aug 29 11:10:31 +0200 2008
|
73
|
+
~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
|
74
|
+
~ {:dispatch_time=>0.010311, :after_filters_time=>8.0e-05, :before_filters_time=>0.003195, :action_time=>0.009567}
|
75
|
+
~
|
76
|
+
|
77
|
+
~ Started request handling: Fri Aug 29 11:10:33 +0200 2008
|
78
|
+
~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
|
79
|
+
~ {:dispatch_time=>0.012913, :after_filters_time=>7.1e-05, :before_filters_time=>0.004422, :action_time=>0.012141}
|
80
|
+
~
|
81
|
+
|
82
|
+
~ Started request handling: Fri Aug 29 11:10:35 +0200 2008
|
83
|
+
~ Params: {"action"=>"new", "namespace"=>"dashboard", "controller"=>"employees"}
|
84
|
+
~ {:dispatch_time=>0.013051, :after_filters_time=>7.8e-05, :before_filters_time=>0.003576, :action_time=>0.011773}
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
|
4
|
+
require "#{File.dirname(__FILE__)}/../lib/merb_analyzer/log_parser"
|
5
|
+
|
6
|
+
class MerbLogParserTest < Test::Unit::TestCase
|
7
|
+
|
8
|
+
def fragment_file(number)
|
9
|
+
"#{File.dirname(__FILE__)}/log_fragments/merb_#{number}.log"
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_parse_started_merb_fragment
|
13
|
+
requests = []
|
14
|
+
parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:started) do |request|
|
15
|
+
requests << request
|
16
|
+
end
|
17
|
+
assert_equal requests[0][:timestamp], "Fri Aug 29 11:10:23 +0200 2008"
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_parse_completed_merb_fragment
|
21
|
+
requests = []
|
22
|
+
parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:completed) do |request|
|
23
|
+
requests << request
|
24
|
+
end
|
25
|
+
|
26
|
+
assert_equal requests[0][:action_time], 0.241652
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_parse_params_merb_fragment
|
30
|
+
requests = []
|
31
|
+
parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:params) do |request|
|
32
|
+
requests << request
|
33
|
+
end
|
34
|
+
|
35
|
+
assert_match '"controller"=>"session"', requests[0][:raw_hash]
|
36
|
+
assert_match '"action"=>"destroy"', requests[0][:raw_hash]
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
@@ -1,17 +1,27 @@
|
|
1
1
|
require 'test/unit'
|
2
2
|
|
3
|
+
require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
|
3
4
|
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
|
4
5
|
|
5
|
-
class
|
6
|
+
class RailsLogParserTest < Test::Unit::TestCase
|
6
7
|
|
7
8
|
def fragment_file(number)
|
8
9
|
"#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
|
9
10
|
end
|
10
11
|
|
11
12
|
|
13
|
+
def test_rails_22_log_format
|
14
|
+
count = 0
|
15
|
+
parser = RailsAnalyzer::LogParser.new(fragment_file(3)).each(:completed) do |request|
|
16
|
+
count += 1
|
17
|
+
assert_equal 0.614, request[:duration] # should be 0.614
|
18
|
+
end
|
19
|
+
assert_equal 1, count
|
20
|
+
end
|
21
|
+
|
12
22
|
def test_progress_messages
|
13
23
|
log_file = fragment_file(1)
|
14
|
-
|
24
|
+
|
15
25
|
finished_encountered = false
|
16
26
|
file_size = File.size(log_file)
|
17
27
|
|
@@ -71,15 +81,15 @@ class LogParserTest < Test::Unit::TestCase
|
|
71
81
|
assert_equal '10.1.1.33', request[:ip]
|
72
82
|
assert_equal '2008-07-13 06:25:58', request[:timestamp]
|
73
83
|
end
|
74
|
-
|
84
|
+
|
75
85
|
parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
|
76
86
|
assert_equal "http://example.com/employee.xml", request[:url]
|
77
87
|
assert_equal 200, request[:status]
|
78
88
|
assert_equal 0.21665, request[:duration]
|
79
|
-
assert_equal 0.00926, request[:rendering]
|
89
|
+
assert_equal 0.00926, request[:rendering]
|
80
90
|
assert_equal 0.0, request[:db]
|
81
91
|
end
|
82
|
-
|
92
|
+
|
83
93
|
end
|
84
94
|
|
85
95
|
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
require 'test/unit'
|
2
2
|
|
3
|
+
require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
|
4
|
+
require "#{File.dirname(__FILE__)}/../lib/base/record_inserter"
|
5
|
+
|
3
6
|
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
|
4
7
|
require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
|
5
8
|
|
data/test/tasks.rake
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wvanbergen-request-log-analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Willem van Bergen
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2008-
|
13
|
+
date: 2008-12-11 00:00:00 -08:00
|
14
14
|
default_executable: request-log-analyzer
|
15
15
|
dependencies: []
|
16
16
|
|
@@ -32,11 +32,18 @@ files:
|
|
32
32
|
- bin/request-log-analyzer
|
33
33
|
- bin/request-log-database
|
34
34
|
- lib
|
35
|
+
- lib/base
|
36
|
+
- lib/base/log_parser.rb
|
37
|
+
- lib/base/record_inserter.rb
|
38
|
+
- lib/base/summarizer.rb
|
35
39
|
- lib/bashcolorizer.rb
|
36
40
|
- lib/command_line
|
37
41
|
- lib/command_line/arguments.rb
|
38
42
|
- lib/command_line/exceptions.rb
|
39
43
|
- lib/command_line/flag.rb
|
44
|
+
- lib/merb_analyzer
|
45
|
+
- lib/merb_analyzer/log_parser.rb
|
46
|
+
- lib/merb_analyzer/summarizer.rb
|
40
47
|
- lib/rails_analyzer
|
41
48
|
- lib/rails_analyzer/log_parser.rb
|
42
49
|
- lib/rails_analyzer/record_inserter.rb
|
@@ -61,7 +68,10 @@ files:
|
|
61
68
|
- test/log_fragments
|
62
69
|
- test/log_fragments/fragment_1.log
|
63
70
|
- test/log_fragments/fragment_2.log
|
64
|
-
- test/
|
71
|
+
- test/log_fragments/fragment_3.log
|
72
|
+
- test/log_fragments/merb_1.log
|
73
|
+
- test/merb_log_parser_test.rb
|
74
|
+
- test/rails_log_parser_test.rb
|
65
75
|
- test/record_inserter_test.rb
|
66
76
|
- test/tasks.rake
|
67
77
|
has_rdoc: false
|
@@ -91,5 +101,6 @@ signing_key:
|
|
91
101
|
specification_version: 2
|
92
102
|
summary: A command line tool to analyze Rails logs
|
93
103
|
test_files:
|
94
|
-
- test/
|
104
|
+
- test/merb_log_parser_test.rb
|
105
|
+
- test/rails_log_parser_test.rb
|
95
106
|
- test/record_inserter_test.rb
|