wvanbergen-request-log-analyzer 1.0.1 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.textile +20 -33
- data/bin/request-log-analyzer +0 -1
- data/lib/request_log_analyzer/aggregator/base.rb +0 -6
- data/lib/request_log_analyzer/aggregator/database.rb +62 -23
- data/lib/request_log_analyzer/aggregator/summarizer.rb +3 -3
- data/lib/request_log_analyzer/controller.rb +21 -26
- data/lib/request_log_analyzer/file_format.rb +4 -0
- data/lib/request_log_analyzer/file_format/rails.rb +7 -36
- data/lib/request_log_analyzer/log_parser.rb +11 -21
- data/lib/request_log_analyzer/request.rb +10 -30
- data/lib/request_log_analyzer/source/base.rb +2 -2
- data/lib/request_log_analyzer/source/log_file.rb +16 -26
- data/spec/database_inserter_spec.rb +32 -15
- data/spec/fixtures/test_file_format.log +3 -1
- data/spec/log_parser_spec.rb +17 -39
- data/spec/merb_format_spec.rb +9 -29
- data/spec/rails_format_spec.rb +7 -26
- data/spec/request_spec.rb +23 -31
- data/spec/summarizer_spec.rb +1 -7
- metadata +2 -2
data/README.textile
CHANGED
@@ -1,49 +1,36 @@
|
|
1
|
-
h1. Request
|
1
|
+
h1. Request-log-analyzer
|
2
2
|
|
3
3
|
This is a simple command line tool to analyze request log files of both Rails and
|
4
|
-
Merb. Its purpose is to find what actions are best candidates for optimization.
|
4
|
+
Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
|
5
5
|
|
6
6
|
* Analyzes Rails log files (all versions)
|
7
7
|
* Can combine multiple files (handy if you are using logrotate)
|
8
8
|
* Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and states, Rails action cache statistics, etc.) ("Sample output":http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
|
9
9
|
* Low memory footprint (server-safe)
|
10
|
-
* MIT licensed
|
11
10
|
* Fast
|
12
|
-
|
13
|
-
h2. Additional information
|
14
|
-
|
15
|
-
* "Project wiki at GitHub":http://wiki.github.com/wvanbergen/request-log-analyzer
|
16
|
-
* "wvanbergen's blog posts":http://techblog.floorplanner.com/tag/request-log-analyzer/
|
11
|
+
* MIT licensed
|
17
12
|
|
18
13
|
h2. Installation
|
19
14
|
|
20
|
-
|
15
|
+
<pre>
|
16
|
+
$ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
|
17
|
+
</pre>
|
18
|
+
|
19
|
+
To get the best results out of request-log-analyzer, make sure to
|
20
|
+
"set up logging correctly":http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
|
21
|
+
for your application.
|
21
22
|
|
22
23
|
h2. Usage
|
23
24
|
|
25
|
+
To analyze a log file and produce a performance report, run request-log-analyzer like this:
|
26
|
+
|
24
27
|
<pre>
|
25
|
-
|
26
|
-
|
27
|
-
Input options:
|
28
|
-
--format <format>, -f: Uses the specified log file format. Defaults to rails.
|
29
|
-
--after <date> Only consider requests from <date> or later.
|
30
|
-
--before <date> Only consider requests before <date>.
|
31
|
-
--select <field> <value> Only consider requests where <field> matches <value>.
|
32
|
-
--reject <field> <value> Only consider requests where <field> does not match <value>.
|
33
|
-
|
34
|
-
Output options:
|
35
|
-
--boring, -b Output reports without ASCII colors.
|
36
|
-
--database <filename>, -d: Creates an SQLite3 database of all the parsed request information.
|
37
|
-
--debug Print debug information while parsing.
|
38
|
-
--file <filename> Output to file.
|
39
|
-
|
40
|
-
Examples:
|
41
|
-
request-log-analyzer development.log
|
42
|
-
request-log-analyzer mongrel.0.log mongrel.1.log mongrel.2.log
|
43
|
-
request-log-analyzer --format merb -d requests.db production.log
|
44
|
-
|
45
|
-
To install rake tasks in your Rails application,
|
46
|
-
run the following command in your application's root directory:
|
47
|
-
|
48
|
-
request-log-analyzer install rails
|
28
|
+
$ request-log-analyzer log/production.log
|
49
29
|
</pre>
|
30
|
+
|
31
|
+
For more details and available command line options, see the "project's wiki":http://wiki.github.com/wvanbergen/request-log-analyzer/basic-usage
|
32
|
+
|
33
|
+
h2. Additional information
|
34
|
+
|
35
|
+
* "Project wiki at GitHub":http://wiki.github.com/wvanbergen/request-log-analyzer
|
36
|
+
* "wvanbergen's blog posts":http://techblog.floorplanner.com/tag/request-log-analyzer/
|
data/bin/request-log-analyzer
CHANGED
@@ -37,7 +37,6 @@ begin
|
|
37
37
|
|
38
38
|
command_line.option(:format, :alias => :f, :default => 'rails')
|
39
39
|
command_line.option(:file, :alias => :e)
|
40
|
-
command_line.switch(:single_lines, :s)
|
41
40
|
command_line.switch(:assume_correct_order)
|
42
41
|
|
43
42
|
command_line.option(:aggregator, :alias => :a, :multiple => true)
|
@@ -2,12 +2,6 @@ module RequestLogAnalyzer::Aggregator
|
|
2
2
|
|
3
3
|
# The base class of an aggregator. This class provides the interface to which
|
4
4
|
# every aggregator should comply (by simply subclassing this class).
|
5
|
-
#
|
6
|
-
# When building an aggregator, do not forget that RequestLogAnalyzer can run in
|
7
|
-
# single line mode or in combined requests mode. Make sure your aggregator can
|
8
|
-
# handle both cases, or raise an exception if RLA is rnning in the wrong mode.
|
9
|
-
# Calling options[:combined_requests] tells you if RLA is running in combined
|
10
|
-
# requests mode, otherwise it is running in single line mode.
|
11
5
|
class Base
|
12
6
|
|
13
7
|
include RequestLogAnalyzer::FileFormat::Awareness
|
@@ -5,43 +5,45 @@ module RequestLogAnalyzer::Aggregator
|
|
5
5
|
|
6
6
|
class Database < Base
|
7
7
|
|
8
|
-
|
9
|
-
|
8
|
+
# Establishes a connection to the database and creates the necessary database schema for the
|
9
|
+
# current file format
|
10
10
|
def prepare
|
11
11
|
ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
|
12
|
-
|
13
|
-
File.unlink(options[:database]) if File.exist?(options[:database])
|
12
|
+
File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
|
14
13
|
create_database_schema!
|
15
|
-
|
16
|
-
@request_id = 0
|
17
14
|
end
|
18
15
|
|
16
|
+
# Aggregates a request into the database
|
17
|
+
# This will create a record in the requests table and create a record for every line that has been parsed,
|
18
|
+
# in which the captured values will be stored.
|
19
19
|
def aggregate(request)
|
20
|
-
@
|
21
|
-
|
20
|
+
@request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
|
22
21
|
request.lines.each do |line|
|
23
|
-
class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
|
24
|
-
|
25
22
|
attributes = line.reject { |k, v| [:line_type].include?(k) }
|
26
|
-
|
27
|
-
file_format.class.const_get(class_name).create!(attributes)
|
23
|
+
@request_object.send("#{line[:line_type]}_lines").build(attributes)
|
28
24
|
end
|
25
|
+
@request_object.save!
|
29
26
|
rescue SQLite3::SQLException => e
|
30
27
|
raise Interrupt, e.message
|
31
28
|
end
|
32
29
|
|
30
|
+
# Finalizes the aggregator by closing the connection to the database
|
33
31
|
def finalize
|
32
|
+
@request_count = @orm_module::Request.count
|
34
33
|
ActiveRecord::Base.remove_connection
|
35
34
|
end
|
36
35
|
|
36
|
+
# Records w warining in the warnings table.
|
37
37
|
def warning(type, message, lineno)
|
38
|
-
|
38
|
+
@orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
|
39
39
|
end
|
40
40
|
|
41
|
+
# Prints a short report of what has been inserted into the database
|
41
42
|
def report(output = STDOUT, report_width = 80, color = false)
|
42
43
|
output << "\n"
|
43
44
|
output << green("━" * report_width, color) + "\n"
|
44
45
|
output << "A database file has been created with all parsed request information.\n"
|
46
|
+
output << "#{@request_count} requests have been added to the database.\n"
|
45
47
|
output << "To execute queries on this database, run the following command:\n"
|
46
48
|
output << " $ sqlite3 #{options[:database]}\n"
|
47
49
|
output << "\n"
|
@@ -49,17 +51,47 @@ module RequestLogAnalyzer::Aggregator
|
|
49
51
|
|
50
52
|
protected
|
51
53
|
|
54
|
+
# This function creates a database table for a given line definition.
|
55
|
+
# It will create a field for every capture in the line, and adds a lineno field to indicate at
|
56
|
+
# what line in the original file the line was found, and a request_id to link lines related
|
57
|
+
# to the same request.
|
52
58
|
def create_database_table(name, definition)
|
53
59
|
ActiveRecord::Migration.verbose = options[:debug]
|
54
60
|
ActiveRecord::Migration.create_table("#{name}_lines") do |t|
|
55
|
-
t.column(:request_id, :integer)
|
61
|
+
t.column(:request_id, :integer)
|
56
62
|
t.column(:lineno, :integer)
|
57
63
|
definition.captures.each do |capture|
|
58
64
|
t.column(capture[:name], column_type(capture))
|
59
65
|
end
|
60
66
|
end
|
61
67
|
end
|
68
|
+
|
69
|
+
# Creates an ActiveRecord class for a given line definition.
|
70
|
+
# A subclass of ActiveRecord::Base is created and an association with the Request class is
|
71
|
+
# created using belongs_to / has_many. This association will later be used to create records
|
72
|
+
# in the corresponding table. This table should already be created before this method is called.
|
73
|
+
def create_activerecord_class(name, definition)
|
74
|
+
class_name = "#{name}_line".camelize
|
75
|
+
klass = Class.new(ActiveRecord::Base)
|
76
|
+
klass.send(:belongs_to, :request)
|
77
|
+
@orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
|
78
|
+
@request_class.send(:has_many, "#{name}_lines".to_sym)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Creates a requests table, in which a record is created for every request. It also creates an
|
82
|
+
# ActiveRecord::Base class to communicate with this table.
|
83
|
+
def create_request_table_and_class
|
84
|
+
ActiveRecord::Migration.verbose = options[:debug]
|
85
|
+
ActiveRecord::Migration.create_table("requests") do |t|
|
86
|
+
t.integer :first_lineno
|
87
|
+
t.integer :last_lineno
|
88
|
+
end
|
89
|
+
|
90
|
+
@orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')
|
91
|
+
@request_class = @orm_module.const_get('Request')
|
92
|
+
end
|
62
93
|
|
94
|
+
# Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
|
63
95
|
def create_warning_table_and_class
|
64
96
|
ActiveRecord::Migration.verbose = options[:debug]
|
65
97
|
ActiveRecord::Migration.create_table("warnings") do |t|
|
@@ -68,23 +100,30 @@ module RequestLogAnalyzer::Aggregator
|
|
68
100
|
t.integer :lineno
|
69
101
|
end
|
70
102
|
|
71
|
-
|
103
|
+
@orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
|
72
104
|
end
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
file_format.class.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?(class_name)
|
77
|
-
end
|
78
|
-
|
105
|
+
|
106
|
+
# Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
|
107
|
+
# file format definition. These ORM classes will later be used to create records in the database.
|
79
108
|
def create_database_schema!
|
109
|
+
|
110
|
+
if file_format.class.const_defined?('Database')
|
111
|
+
@orm_module = file_format.class.const_get('Database')
|
112
|
+
else
|
113
|
+
@orm_module = file_format.class.const_set('Database', Module.new)
|
114
|
+
end
|
115
|
+
|
116
|
+
create_request_table_and_class
|
117
|
+
create_warning_table_and_class
|
118
|
+
|
80
119
|
file_format.line_definitions.each do |name, definition|
|
81
120
|
create_database_table(name, definition)
|
82
121
|
create_activerecord_class(name, definition)
|
83
122
|
end
|
84
|
-
|
85
|
-
create_warning_table_and_class
|
86
123
|
end
|
87
124
|
|
125
|
+
# Function to determine the column type for a field
|
126
|
+
# TODO: make more robust / include in file-format definition
|
88
127
|
def column_type(capture)
|
89
128
|
case capture[:type]
|
90
129
|
when :sec; :double
|
@@ -69,7 +69,7 @@ module RequestLogAnalyzer::Aggregator
|
|
69
69
|
|
70
70
|
def report(output=STDOUT, report_width = 80, color = false)
|
71
71
|
report_header(output, report_width, color)
|
72
|
-
if source.parsed_requests
|
72
|
+
if source.parsed_requests > 0
|
73
73
|
@trackers.each { |tracker| tracker.report(output, report_width, color) }
|
74
74
|
else
|
75
75
|
output << "\n"
|
@@ -82,8 +82,8 @@ module RequestLogAnalyzer::Aggregator
|
|
82
82
|
output << "Request summary\n"
|
83
83
|
output << green("━" * report_width, color) + "\n"
|
84
84
|
output << "Parsed lines: #{green(source.parsed_lines, color)}\n"
|
85
|
-
output << "Parsed requests: #{green(source.parsed_requests, color)}\n"
|
86
|
-
output << "Skipped
|
85
|
+
output << "Parsed requests: #{green(source.parsed_requests, color)}\n"
|
86
|
+
output << "Skipped lines: #{green(source.skipped_lines, color)}\n" if source.skipped_lines > 0
|
87
87
|
if has_warnings?
|
88
88
|
output << "Warnings: " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
|
89
89
|
end
|
@@ -33,7 +33,6 @@ module RequestLogAnalyzer
|
|
33
33
|
|
34
34
|
options = { :report_width => arguments[:report_width].to_i, :output => STDOUT}
|
35
35
|
|
36
|
-
options[:combined_requests] = !arguments[:single_lines]
|
37
36
|
options[:database] = arguments[:database] if arguments[:database]
|
38
37
|
options[:debug] = arguments[:debug]
|
39
38
|
options[:colorize] = !arguments[:boring]
|
@@ -45,7 +44,7 @@ module RequestLogAnalyzer
|
|
45
44
|
|
46
45
|
# Create the controller with the correct file format
|
47
46
|
file_format = RequestLogAnalyzer::FileFormat.load(arguments[:format])
|
48
|
-
|
47
|
+
|
49
48
|
# register sources
|
50
49
|
if arguments.parameters.length == 1
|
51
50
|
file = arguments.parameters[0]
|
@@ -63,27 +62,22 @@ module RequestLogAnalyzer
|
|
63
62
|
|
64
63
|
controller = Controller.new(RequestLogAnalyzer::Source::LogFile.new(file_format, options), options)
|
65
64
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
arguments[:select].each do |(field, value)|
|
84
|
-
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
85
|
-
end
|
86
|
-
|
65
|
+
options[:assume_correct_order] = arguments[:assume_correct_order]
|
66
|
+
|
67
|
+
# register filters
|
68
|
+
if arguments[:after] || arguments[:before]
|
69
|
+
filter_options = {}
|
70
|
+
filter_options[:after] = DateTime.parse(arguments[:after])
|
71
|
+
filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
|
72
|
+
controller.add_filter(:timespan, filter_options)
|
73
|
+
end
|
74
|
+
|
75
|
+
arguments[:reject].each do |(field, value)|
|
76
|
+
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
|
77
|
+
end
|
78
|
+
|
79
|
+
arguments[:select].each do |(field, value)|
|
80
|
+
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
87
81
|
end
|
88
82
|
|
89
83
|
# register aggregators
|
@@ -95,7 +89,9 @@ module RequestLogAnalyzer
|
|
95
89
|
|
96
90
|
# register the echo aggregator in debug mode
|
97
91
|
controller.add_aggregator(:echo) if arguments[:debug]
|
98
|
-
|
92
|
+
|
93
|
+
file_format.setup_environment(controller)
|
94
|
+
|
99
95
|
return controller
|
100
96
|
end
|
101
97
|
|
@@ -103,7 +99,6 @@ module RequestLogAnalyzer
|
|
103
99
|
# <tt>format</tt> Logfile format. Defaults to :rails
|
104
100
|
# Options are passd on to the LogParser.
|
105
101
|
# * <tt>:aggregator</tt> Aggregator array.
|
106
|
-
# * <tt>:combined_requests</tt> Combine multiline requests into a single request.
|
107
102
|
# * <tt>:database</tt> Database the controller should use.
|
108
103
|
# * <tt>:echo</tt> Output debug information.
|
109
104
|
# * <tt>:silent</tt> Do not output any warnings.
|
@@ -186,7 +181,7 @@ module RequestLogAnalyzer
|
|
186
181
|
|
187
182
|
begin
|
188
183
|
@source.requests do |request|
|
189
|
-
|
184
|
+
#@filters.each { |filter| request = filter.filter(request) }
|
190
185
|
@aggregators.each { |agg| agg.aggregate(request) } if request
|
191
186
|
end
|
192
187
|
rescue Interrupt => e
|
@@ -22,11 +22,11 @@ class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
|
|
22
22
|
line_definition :failed do |line|
|
23
23
|
line.footer = true
|
24
24
|
line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
|
25
|
-
line.captures << { :name => :error,
|
26
|
-
<< { :name => :
|
27
|
-
<< { :name => :line,
|
28
|
-
<< { :name => :file,
|
29
|
-
<< { :name => :stack_trace,
|
25
|
+
line.captures << { :name => :error, :type => :string } \
|
26
|
+
<< { :name => :message, :type => :string } \
|
27
|
+
<< { :name => :line, :type => :integer } \
|
28
|
+
<< { :name => :file, :type => :string } \
|
29
|
+
<< { :name => :stack_trace, :type => :string, :anonymize => true }
|
30
30
|
end
|
31
31
|
|
32
32
|
|
@@ -63,37 +63,8 @@ class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
|
|
63
63
|
|
64
64
|
|
65
65
|
REQUEST_CATEGORIZER = Proc.new do |request|
|
66
|
-
|
67
|
-
|
68
|
-
if request =~ :failed
|
69
|
-
format = request[:format] || 'html'
|
70
|
-
"#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
71
|
-
else
|
72
|
-
format = request[:format] || 'html'
|
73
|
-
"#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
74
|
-
end
|
75
|
-
|
76
|
-
else
|
77
|
-
case request.line_type
|
78
|
-
when :processing
|
79
|
-
format = request[:format] || 'html'
|
80
|
-
"#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
81
|
-
|
82
|
-
when :completed
|
83
|
-
url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
|
84
|
-
url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
|
85
|
-
|
86
|
-
url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date/') # Combine all (year-month-day) queries
|
87
|
-
url.gsub!(/\/\d+-\d+(\/|$)/, '/:month/') # Combine all date (year-month) queries
|
88
|
-
url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
|
89
|
-
url
|
90
|
-
|
91
|
-
when :failed
|
92
|
-
request[:error]
|
93
|
-
else
|
94
|
-
raise "Cannot group this request: #{request.inspect}"
|
95
|
-
end
|
96
|
-
end
|
66
|
+
format = request[:format] || 'html'
|
67
|
+
"#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
97
68
|
end
|
98
69
|
|
99
70
|
report do |analyze|
|
@@ -1,18 +1,14 @@
|
|
1
1
|
module RequestLogAnalyzer
|
2
2
|
|
3
3
|
# The LogParser class reads log data from a given source and uses a file format definition
|
4
|
-
# to parse all relevent information about requests from the file.
|
4
|
+
# to parse all relevent information about requests from the file. A FileFormat module should
|
5
|
+
# be provided that contains the definitions of the lines that occur in the log data.
|
5
6
|
#
|
6
|
-
#
|
7
|
-
#
|
8
|
-
#
|
9
|
-
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
# The combined requests mode gives better information, but can be problematic if the log
|
13
|
-
# file is unordered. This can be the case if data is written to the log file simultaneously
|
14
|
-
# by different mongrel processes. This problem is detected by the parser, but the requests
|
15
|
-
# that are mixed up cannot be parsed. It will emit warnings when this occurs.
|
7
|
+
# De order in which lines occur is used to combine lines to a single request. If these lines
|
8
|
+
# are mixed, requests cannot be combined properly. This can be the case if data is written to
|
9
|
+
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
|
+
# parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this
|
11
|
+
# occurs.
|
16
12
|
class LogParser
|
17
13
|
|
18
14
|
include RequestLogAnalyzer::FileFormat::Awareness
|
@@ -79,8 +75,6 @@ module RequestLogAnalyzer
|
|
79
75
|
unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
|
80
76
|
raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
|
81
77
|
|
82
|
-
puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
|
83
|
-
|
84
78
|
@current_io = io
|
85
79
|
@current_io.each_line do |line|
|
86
80
|
|
@@ -94,11 +88,7 @@ module RequestLogAnalyzer
|
|
94
88
|
|
95
89
|
if request_data
|
96
90
|
@parsed_lines += 1
|
97
|
-
|
98
|
-
update_current_request(request_data, &block)
|
99
|
-
else
|
100
|
-
handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
|
101
|
-
end
|
91
|
+
update_current_request(request_data, &block)
|
102
92
|
end
|
103
93
|
end
|
104
94
|
|
@@ -126,9 +116,9 @@ module RequestLogAnalyzer
|
|
126
116
|
|
127
117
|
protected
|
128
118
|
|
129
|
-
# Combines the different lines of a request into a single Request object.
|
130
|
-
#
|
131
|
-
#
|
119
|
+
# Combines the different lines of a request into a single Request object. It will start a
|
120
|
+
# new request when a header line is encountered en will emit the request when a footer line
|
121
|
+
# is encountered.
|
132
122
|
#
|
133
123
|
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
134
124
|
# any request. It will emit a :no_current_request warning.
|
@@ -4,12 +4,6 @@ module RequestLogAnalyzer
|
|
4
4
|
# Instances are created by the LogParser and are passed to the different aggregators, so they
|
5
5
|
# can do their aggregating work.
|
6
6
|
#
|
7
|
-
# Note that RequestLogAnalyzer can run in two modes:
|
8
|
-
# - Single line mode: every parsed line is regarded as a request. Request::single_line? will
|
9
|
-
# return true in this case
|
10
|
-
# - Combined requests mode: lines that belong together are grouped into one request.
|
11
|
-
# Request#combined? will return true in this case.
|
12
|
-
#
|
13
7
|
# This class provides several methods to access the data that was parsed from the log files.
|
14
8
|
# Request#first(field_name) returns the first (only) value corresponding to the given field
|
15
9
|
# Request#every(field_name) returns all values corresponding to the given field name as array.
|
@@ -54,7 +48,7 @@ module RequestLogAnalyzer
|
|
54
48
|
|
55
49
|
# Returns the value that was captured for the "field" of this request.
|
56
50
|
# This function will return the first value that was captured if the field
|
57
|
-
# was captured in multiple lines
|
51
|
+
# was captured in multiple lines
|
58
52
|
def first(field)
|
59
53
|
@attributes[field]
|
60
54
|
end
|
@@ -72,44 +66,30 @@ module RequestLogAnalyzer
|
|
72
66
|
@lines.length == 0
|
73
67
|
end
|
74
68
|
|
75
|
-
# Checks whether this request contains exactly one line. This means that RequestLogAnalyzer
|
76
|
-
# is running in single_line mode.
|
77
|
-
def single_line?
|
78
|
-
@lines.length == 1
|
79
|
-
end
|
80
|
-
|
81
|
-
# Checks whether this request contains more than one line. This means that RequestLogAnalyzer
|
82
|
-
# is runring in combined requests mode.
|
83
|
-
def combined?
|
84
|
-
@lines.length > 1
|
85
|
-
end
|
86
|
-
|
87
69
|
# Checks whether this request is completed. A completed request contains both a parsed header
|
88
70
|
# line and a parsed footer line. Not that calling this function in single line mode will always
|
89
71
|
# return false.
|
90
72
|
def completed?
|
91
|
-
puts attributes[:method]
|
92
|
-
|
93
73
|
header_found, footer_found = false, false
|
94
74
|
@lines.each do |line|
|
95
75
|
line_def = file_format.line_definitions[line[:line_type]]
|
96
76
|
header_found = true if line_def.header
|
97
77
|
footer_found = true if line_def.footer
|
98
78
|
end
|
99
|
-
header_found && footer_found
|
100
|
-
|
101
|
-
end
|
102
|
-
|
103
|
-
# Returns the line type of the parsed line of this request.
|
104
|
-
# This function can only be called in single line mode.
|
105
|
-
def line_type
|
106
|
-
raise "Not a single line request!" unless single_line?
|
107
|
-
lines.first[:line_type]
|
79
|
+
header_found && footer_found
|
108
80
|
end
|
109
81
|
|
110
82
|
# Returns the first timestamp encountered in a request.
|
111
83
|
def timestamp
|
112
84
|
first(:timestamp)
|
113
85
|
end
|
86
|
+
|
87
|
+
def first_lineno
|
88
|
+
@lines.first[:lineno]
|
89
|
+
end
|
90
|
+
|
91
|
+
def last_lineno
|
92
|
+
@lines.last[:lineno]
|
93
|
+
end
|
114
94
|
end
|
115
95
|
end
|
@@ -15,8 +15,8 @@ module RequestLogAnalyzer::Source
|
|
15
15
|
# The total number of parsed requests.
|
16
16
|
attr_reader :parsed_requests
|
17
17
|
|
18
|
-
# The number of skipped
|
19
|
-
attr_reader :
|
18
|
+
# The number of skipped lines because of warnings
|
19
|
+
attr_reader :skipped_lines
|
20
20
|
|
21
21
|
# Base source class used to filter input requests.
|
22
22
|
|
@@ -1,17 +1,14 @@
|
|
1
1
|
module RequestLogAnalyzer::Source
|
2
|
+
|
2
3
|
# The LogParser class reads log data from a given source and uses a file format definition
|
3
|
-
# to parse all relevent information about requests from the file.
|
4
|
+
# to parse all relevent information about requests from the file. A FileFormat module should
|
5
|
+
# be provided that contains the definitions of the lines that occur in the log data.
|
4
6
|
#
|
5
|
-
#
|
6
|
-
#
|
7
|
-
#
|
8
|
-
#
|
9
|
-
#
|
10
|
-
#
|
11
|
-
# The combined requests mode gives better information, but can be problematic if the log
|
12
|
-
# file is unordered. This can be the case if data is written to the log file simultaneously
|
13
|
-
# by different mongrel processes. This problem is detected by the parser, but the requests
|
14
|
-
# that are mixed up cannot be parsed. It will emit warnings when this occurs.
|
7
|
+
# De order in which lines occur is used to combine lines to a single request. If these lines
|
8
|
+
# are mixed, requests cannot be combined properly. This can be the case if data is written to
|
9
|
+
# the log file simultaneously by different mongrel processes. This problem is detected by the
|
10
|
+
# parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this
|
11
|
+
# occurs.
|
15
12
|
class LogFile < RequestLogAnalyzer::Source::Base
|
16
13
|
|
17
14
|
attr_reader :source_files
|
@@ -24,8 +21,8 @@ module RequestLogAnalyzer::Source
|
|
24
21
|
@options = options
|
25
22
|
@parsed_lines = 0
|
26
23
|
@parsed_requests = 0
|
27
|
-
@
|
28
|
-
@current_io
|
24
|
+
@skipped_lines = 0
|
25
|
+
@current_io = nil
|
29
26
|
@source_files = options[:source_files]
|
30
27
|
|
31
28
|
# install the file format module (see RequestLogAnalyzer::FileFormat)
|
@@ -78,8 +75,6 @@ module RequestLogAnalyzer::Source
|
|
78
75
|
unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
|
79
76
|
raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
|
80
77
|
|
81
|
-
puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
|
82
|
-
|
83
78
|
@current_io = io
|
84
79
|
@current_io.each_line do |line|
|
85
80
|
|
@@ -93,12 +88,7 @@ module RequestLogAnalyzer::Source
|
|
93
88
|
|
94
89
|
if request_data
|
95
90
|
@parsed_lines += 1
|
96
|
-
|
97
|
-
update_current_request(request_data, &block)
|
98
|
-
else
|
99
|
-
@parsed_requests +=1
|
100
|
-
yield RequestLogAnalyzer::Request.create(@file_format, request_data)
|
101
|
-
end
|
91
|
+
update_current_request(request_data, &block)
|
102
92
|
end
|
103
93
|
end
|
104
94
|
|
@@ -126,9 +116,9 @@ module RequestLogAnalyzer::Source
|
|
126
116
|
|
127
117
|
protected
|
128
118
|
|
129
|
-
# Combines the different lines of a request into a single Request object.
|
130
|
-
#
|
131
|
-
#
|
119
|
+
# Combines the different lines of a request into a single Request object. It will start a
|
120
|
+
# new request when a header line is encountered en will emit the request when a footer line
|
121
|
+
# is encountered.
|
132
122
|
#
|
133
123
|
# - Every line that is parsed before a header line is ignored as it cannot be included in
|
134
124
|
# any request. It will emit a :no_current_request warning.
|
@@ -144,7 +134,7 @@ module RequestLogAnalyzer::Source
|
|
144
134
|
yield @current_request
|
145
135
|
@current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
|
146
136
|
else
|
147
|
-
@
|
137
|
+
@skipped_lines += 1
|
148
138
|
warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
|
149
139
|
@current_request = nil # remove all data that was parsed, skip next request as well.
|
150
140
|
end
|
@@ -160,7 +150,7 @@ module RequestLogAnalyzer::Source
|
|
160
150
|
@current_request = nil
|
161
151
|
end
|
162
152
|
else
|
163
|
-
@
|
153
|
+
@skipped_lines += 1
|
164
154
|
warn(:no_current_request, "Parsebale line found outside of a request!")
|
165
155
|
end
|
166
156
|
end
|
@@ -18,16 +18,31 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
18
18
|
|
19
19
|
it "should create the correct tables" do
|
20
20
|
ActiveRecord::Migration.should_receive(:create_table).with("warnings")
|
21
|
+
ActiveRecord::Migration.should_receive(:create_table).with("requests")
|
21
22
|
ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
|
22
23
|
ActiveRecord::Migration.should_receive(:create_table).with("test_lines")
|
23
24
|
ActiveRecord::Migration.should_receive(:create_table).with("last_lines")
|
24
25
|
@database_inserter.prepare
|
25
26
|
end
|
26
27
|
|
28
|
+
it "should create a default Request class" do
|
29
|
+
@database_inserter.prepare
|
30
|
+
SpecFormat::Database::Request.ancestors.should include(ActiveRecord::Base)
|
31
|
+
SpecFormat::Database::Request.column_names.should include('first_lineno')
|
32
|
+
SpecFormat::Database::Request.column_names.should include('last_lineno')
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should create associations for the default Request class" do
|
36
|
+
@database_inserter.prepare
|
37
|
+
@request = SpecFormat::Database::Request.new
|
38
|
+
@request.should respond_to(:test_lines)
|
39
|
+
@request.test_lines.should
|
40
|
+
end
|
41
|
+
|
27
42
|
it "should create the default table names" do
|
28
43
|
@database_inserter.prepare
|
29
44
|
@database_inserter.file_format.line_definitions.each do |name, definition|
|
30
|
-
klass = SpecFormat.const_get("#{name}_line".camelize)
|
45
|
+
klass = SpecFormat::Database.const_get("#{name}_line".camelize)
|
31
46
|
klass.column_names.should include('id')
|
32
47
|
klass.column_names.should include('lineno')
|
33
48
|
klass.column_names.should include('request_id')
|
@@ -37,9 +52,9 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
|
|
37
52
|
it "should create the correct fields in the table" do
|
38
53
|
@database_inserter.prepare
|
39
54
|
|
40
|
-
SpecFormat::FirstLine.column_names.should include('request_no')
|
41
|
-
SpecFormat::LastLine.column_names.should include('request_no')
|
42
|
-
SpecFormat::TestLine.column_names.should include('test_capture')
|
55
|
+
SpecFormat::Database::FirstLine.column_names.should include('request_no')
|
56
|
+
SpecFormat::Database::LastLine.column_names.should include('request_no')
|
57
|
+
SpecFormat::Database::TestLine.column_names.should include('test_capture')
|
43
58
|
end
|
44
59
|
|
45
60
|
end
|
@@ -52,8 +67,8 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
|
52
67
|
@database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
|
53
68
|
@database_inserter.prepare
|
54
69
|
|
55
|
-
@
|
56
|
-
@
|
70
|
+
@incomplete_request = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
|
71
|
+
@completed_request = RequestLogAnalyzer::Request.create(spec_format,
|
57
72
|
{:line_type => :first, :request_no => 564},
|
58
73
|
{:line_type => :test, :test_capture => "awesome"},
|
59
74
|
{:line_type => :test, :test_capture => "indeed"},
|
@@ -64,20 +79,22 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
|
|
64
79
|
File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
|
65
80
|
end
|
66
81
|
|
67
|
-
it "should insert a record in the
|
68
|
-
SpecFormat::
|
69
|
-
@database_inserter.aggregate(@
|
82
|
+
it "should insert a record in the request table" do
|
83
|
+
SpecFormat::Database::Request.count.should == 0
|
84
|
+
@database_inserter.aggregate(@incomplete_request)
|
85
|
+
SpecFormat::Database::Request.count.should == 1
|
70
86
|
end
|
71
87
|
|
72
|
-
it "should insert records in all relevant tables" do
|
73
|
-
|
74
|
-
SpecFormat::
|
75
|
-
|
76
|
-
|
88
|
+
it "should insert records in all relevant line tables" do
|
89
|
+
@database_inserter.aggregate(@completed_request)
|
90
|
+
request = SpecFormat::Database::Request.first
|
91
|
+
request.should have(2).test_lines
|
92
|
+
request.should have(1).first_lines
|
93
|
+
request.should have(1).last_lines
|
77
94
|
end
|
78
95
|
|
79
96
|
it "should log a warning in the warnings table" do
|
80
|
-
SpecFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
|
97
|
+
SpecFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
|
81
98
|
@database_inserter.warning(:test_warning, "Testing the warning system", 12)
|
82
99
|
end
|
83
100
|
|
data/spec/log_parser_spec.rb
CHANGED
@@ -1,46 +1,12 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
|
-
describe RequestLogAnalyzer::LogParser, :
|
4
|
-
|
3
|
+
describe RequestLogAnalyzer::LogParser, :requests do
|
5
4
|
include RequestLogAnalyzerSpecHelper
|
6
5
|
|
7
6
|
before(:each) do
|
8
7
|
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
9
8
|
end
|
10
9
|
|
11
|
-
it "should have line definitions" do
|
12
|
-
@log_parser.file_format.line_definitions.should_not be_empty
|
13
|
-
end
|
14
|
-
|
15
|
-
# it "should have include the language specific hooks in the instance, not in the class" do
|
16
|
-
# metaclass = (class << @log_parser; self; end)
|
17
|
-
# metaclass.ancestors.should include(TestFileFormat::LogParser)
|
18
|
-
# @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
|
19
|
-
# end
|
20
|
-
|
21
|
-
it "should parse a stream and find valid requests" do
|
22
|
-
io = File.new(log_fixture(:test_file_format), 'r')
|
23
|
-
@log_parser.parse_io(io, :line_types => [:test]) do |request|
|
24
|
-
request.should be_kind_of(RequestLogAnalyzer::Request)
|
25
|
-
request.should =~ :test
|
26
|
-
request[:test_capture].should_not be_nil
|
27
|
-
end
|
28
|
-
io.close
|
29
|
-
end
|
30
|
-
|
31
|
-
it "should find as many lines as request" do
|
32
|
-
@log_parser.parse_file(log_fixture(:test_file_format)) { |request| request.should be_single_line }
|
33
|
-
@log_parser.parsed_lines.should eql(@log_parser.parsed_requests)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
describe RequestLogAnalyzer::LogParser, :combined_requests do
|
38
|
-
include RequestLogAnalyzerSpecHelper
|
39
|
-
|
40
|
-
before(:each) do
|
41
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
42
|
-
end
|
43
|
-
|
44
10
|
it "should have multiple line definitions" do
|
45
11
|
@log_parser.file_format.line_definitions.length.should >= 2
|
46
12
|
end
|
@@ -51,7 +17,7 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
|
|
51
17
|
|
52
18
|
it "should parse more lines than requests" do
|
53
19
|
@log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
|
54
|
-
@log_parser.parse_file(log_fixture(:test_language_combined))
|
20
|
+
@log_parser.parse_file(log_fixture(:test_language_combined))
|
55
21
|
@log_parser.parsed_lines.should > 2
|
56
22
|
end
|
57
23
|
|
@@ -68,20 +34,32 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
|
|
68
34
|
request[:test_capture].should == "amazing"
|
69
35
|
end
|
70
36
|
end
|
37
|
+
|
38
|
+
it "should parse a stream and find valid requests" do
|
39
|
+
io = File.new(log_fixture(:test_file_format), 'r')
|
40
|
+
@log_parser.parse_io(io) do |request|
|
41
|
+
request.should be_kind_of(RequestLogAnalyzer::Request)
|
42
|
+
request.should =~ :test
|
43
|
+
request[:test_capture].should_not be_nil
|
44
|
+
end
|
45
|
+
io.close
|
46
|
+
end
|
47
|
+
|
71
48
|
end
|
72
49
|
|
73
50
|
describe RequestLogAnalyzer::LogParser, :warnings do
|
74
51
|
include RequestLogAnalyzerSpecHelper
|
52
|
+
|
53
|
+
before(:each) do
|
54
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
55
|
+
end
|
75
56
|
|
76
57
|
it "should warn about teaser matching problems" do
|
77
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
78
58
|
@log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
|
79
59
|
@log_parser.parse_file(log_fixture(:test_file_format))
|
80
60
|
end
|
81
61
|
|
82
62
|
it "should warn about unmatching request headers and footers" do
|
83
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
|
84
|
-
|
85
63
|
@log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
|
86
64
|
@log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
|
87
65
|
@log_parser.should_not_receive(:handle_request)
|
data/spec/merb_format_spec.rb
CHANGED
@@ -1,45 +1,25 @@
|
|
1
1
|
# require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
#
|
3
|
-
# describe RequestLogAnalyzer::LogParser, "Merb
|
3
|
+
# describe RequestLogAnalyzer::LogParser, "Merb" do
|
4
4
|
# include RequestLogAnalyzerSpecHelper
|
5
5
|
#
|
6
6
|
# before(:each) do
|
7
|
-
# @log_parser = RequestLogAnalyzer::LogParser.new(:merb
|
7
|
+
# @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
|
8
|
+
# end
|
9
|
+
#
|
10
|
+
# it "should have a valid language definitions" do
|
11
|
+
# @log_parser.file_format.should be_valid
|
8
12
|
# end
|
9
13
|
#
|
10
14
|
# it "should parse a stream and find valid requests" do
|
11
15
|
# File.open(log_fixture(:merb), 'r') do |io|
|
12
16
|
# @log_parser.parse_io(io) do |request|
|
13
17
|
# request.should be_kind_of(RequestLogAnalyzer::Request)
|
14
|
-
# request.should be_single_line
|
15
18
|
# end
|
16
19
|
# end
|
17
20
|
# end
|
18
|
-
#
|
19
|
-
# it "should find
|
20
|
-
# @log_parser.should_receive(:handle_request).exactly(33).times
|
21
|
-
# @log_parser.parse_file(log_fixture(:merb))
|
22
|
-
# end
|
23
|
-
#
|
24
|
-
# it "should find 11 request start lines when lines are not linked" do
|
25
|
-
# @log_parser.should_receive(:handle_request).exactly(11).times
|
26
|
-
# @log_parser.parse_file(log_fixture(:merb), :line_types => [:started])
|
27
|
-
# end
|
28
|
-
# end
|
29
|
-
#
|
30
|
-
#
|
31
|
-
# describe RequestLogAnalyzer::LogParser, "Merb with combined requests" do
|
32
|
-
# include RequestLogAnalyzerSpecHelper
|
33
|
-
#
|
34
|
-
# before(:each) do
|
35
|
-
# @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => true)
|
36
|
-
# end
|
37
|
-
#
|
38
|
-
# it "should have a valid language definitions" do
|
39
|
-
# @log_parser.file_format.should be_valid
|
40
|
-
# end
|
41
|
-
#
|
42
|
-
# it "should find 11 completed requests when lines are linked" do
|
21
|
+
#
|
22
|
+
# it "should find 11 completed requests" do
|
43
23
|
# @log_parser.should_receive(:handle_request).exactly(11).times
|
44
24
|
# @log_parser.parse_file(log_fixture(:merb))
|
45
25
|
# end
|
@@ -55,4 +35,4 @@
|
|
55
35
|
# request[:before_filters_time].should == 0.213213
|
56
36
|
# request[:action_time].should == 0.241652
|
57
37
|
# end
|
58
|
-
# end
|
38
|
+
# end
|
data/spec/rails_format_spec.rb
CHANGED
@@ -1,10 +1,14 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
|
-
describe RequestLogAnalyzer::LogParser, "Rails
|
3
|
+
describe RequestLogAnalyzer::LogParser, "Rails" do
|
4
4
|
include RequestLogAnalyzerSpecHelper
|
5
5
|
|
6
6
|
before(:each) do
|
7
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails)
|
7
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
|
8
|
+
end
|
9
|
+
|
10
|
+
it "should have a valid language definitions" do
|
11
|
+
@log_parser.file_format.should be_valid
|
8
12
|
end
|
9
13
|
|
10
14
|
it "should parse a stream and find valid requests" do
|
@@ -13,30 +17,9 @@ describe RequestLogAnalyzer::LogParser, "Rails without combined requests" do
|
|
13
17
|
request.should be_kind_of(RequestLogAnalyzer::Request)
|
14
18
|
end
|
15
19
|
io.close
|
16
|
-
end
|
17
|
-
|
18
|
-
it "should find 8 requests when lines are not linked" do
|
19
|
-
requests = []
|
20
|
-
@log_parser.parse_file(log_fixture(:rails_1x)) { |request| requests << request }
|
21
|
-
requests.length.should == 8
|
22
|
-
requests.each { |r| r.should be_single_line }
|
23
|
-
requests.select { |r| r.line_type == :processing }.should have(4).items
|
24
20
|
end
|
25
|
-
end
|
26
|
-
|
27
|
-
|
28
|
-
describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
|
29
|
-
include RequestLogAnalyzerSpecHelper
|
30
|
-
|
31
|
-
before(:each) do
|
32
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails), :combined_requests => true)
|
33
|
-
end
|
34
|
-
|
35
|
-
it "should have a valid language definitions" do
|
36
|
-
@log_parser.file_format.should be_valid
|
37
|
-
end
|
38
21
|
|
39
|
-
it "should find 4 completed requests
|
22
|
+
it "should find 4 completed requests" do
|
40
23
|
@log_parser.should_not_receive(:warn)
|
41
24
|
@log_parser.should_receive(:handle_request).exactly(4).times
|
42
25
|
@log_parser.parse_file(log_fixture(:rails_1x))
|
@@ -63,7 +46,6 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
|
|
63
46
|
@log_parser.parse_file(log_fixture(:syslog_1x)) do |request|
|
64
47
|
|
65
48
|
request.should be_completed
|
66
|
-
request.should be_combined
|
67
49
|
|
68
50
|
request[:controller].should == 'EmployeeController'
|
69
51
|
request[:action].should == 'index'
|
@@ -79,7 +61,6 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
|
|
79
61
|
@log_parser.should_not_receive(:warn)
|
80
62
|
@log_parser.parse_file(log_fixture(:rails_22_cached)) do |request|
|
81
63
|
request.should be_completed
|
82
|
-
request.should be_combined
|
83
64
|
request =~ :cache_hit
|
84
65
|
end
|
85
66
|
end
|
data/spec/request_spec.rb
CHANGED
@@ -1,80 +1,72 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
|
3
|
-
describe RequestLogAnalyzer::Request, :
|
3
|
+
describe RequestLogAnalyzer::Request, :incomplete_request do
|
4
4
|
|
5
5
|
include RequestLogAnalyzerSpecHelper
|
6
6
|
|
7
7
|
before(:each) do
|
8
|
-
@
|
9
|
-
@
|
8
|
+
@incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
|
9
|
+
@incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
|
10
10
|
end
|
11
11
|
|
12
|
-
it "should include the file format module" do
|
13
|
-
(class << @single_line_request; self; end).ancestors.include?(TestFileFormat)
|
14
|
-
end
|
15
|
-
|
16
12
|
it "should be single if only one line has been added" do
|
17
|
-
@
|
18
|
-
@single_line_request.should_not be_empty
|
19
|
-
@single_line_request.should_not be_combined
|
13
|
+
@incomplete_request.should_not be_empty
|
20
14
|
end
|
21
15
|
|
22
16
|
it "should not be a completed request" do
|
23
|
-
@
|
17
|
+
@incomplete_request.should_not be_completed
|
24
18
|
end
|
25
19
|
|
26
20
|
it "should take the line type of the first line as global line_type" do
|
27
|
-
@
|
28
|
-
@
|
21
|
+
@incomplete_request.lines[0][:line_type].should == :test
|
22
|
+
@incomplete_request.should =~ :test
|
29
23
|
end
|
30
24
|
|
31
25
|
it "should return the first field value" do
|
32
|
-
@
|
26
|
+
@incomplete_request[:test_capture].should == 'awesome!'
|
33
27
|
end
|
34
28
|
|
35
29
|
it "should return nil if no such field is present" do
|
36
|
-
@
|
30
|
+
@incomplete_request[:nonexisting].should be_nil
|
37
31
|
end
|
38
32
|
end
|
39
33
|
|
40
34
|
|
41
|
-
describe RequestLogAnalyzer::Request, :
|
35
|
+
describe RequestLogAnalyzer::Request, :completed_request do
|
42
36
|
|
43
37
|
include RequestLogAnalyzerSpecHelper
|
44
38
|
|
45
39
|
before(:each) do
|
46
|
-
@
|
47
|
-
@
|
48
|
-
@
|
49
|
-
@
|
50
|
-
@
|
40
|
+
@completed_request = RequestLogAnalyzer::Request.new(spec_format)
|
41
|
+
@completed_request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
|
42
|
+
@completed_request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
|
43
|
+
@completed_request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
|
44
|
+
@completed_request << { :line_type => :last, :lineno => 10, :time => 0.03 }
|
51
45
|
end
|
52
46
|
|
53
|
-
it "should be
|
54
|
-
@
|
55
|
-
@combined_request.should_not be_single_line
|
56
|
-
@combined_request.should_not be_empty
|
47
|
+
it "should not be empty when multiple liness are added" do
|
48
|
+
@completed_request.should_not be_empty
|
57
49
|
end
|
58
50
|
|
59
51
|
it "should be a completed request" do
|
60
|
-
@
|
52
|
+
@completed_request.should be_completed
|
61
53
|
end
|
62
54
|
|
63
55
|
it "should recognize all line types" do
|
64
|
-
[:first, :test, :last].each { |type| @
|
56
|
+
[:first, :test, :last].each { |type| @completed_request.should =~ type }
|
65
57
|
end
|
66
58
|
|
67
59
|
it "should detect the correct field value" do
|
68
|
-
@
|
69
|
-
@
|
60
|
+
@completed_request[:name].should == 'first line!'
|
61
|
+
@completed_request[:time].should == 0.03
|
70
62
|
end
|
71
63
|
|
72
64
|
it "should detect the first matching field value" do
|
73
|
-
@
|
65
|
+
@completed_request.first(:test_capture).should == 'testing'
|
74
66
|
end
|
75
67
|
|
76
68
|
it "should detect the every matching field value" do
|
77
|
-
@
|
69
|
+
@completed_request.every(:test_capture).should == ['testing', "testing some more"]
|
78
70
|
end
|
79
71
|
|
80
72
|
end
|
data/spec/summarizer_spec.rb
CHANGED
@@ -1,13 +1,7 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
require 'request_log_analyzer/aggregator/summarizer'
|
3
3
|
|
4
|
-
describe RequestLogAnalyzer::Aggregator::Summarizer
|
5
|
-
|
6
|
-
before(:each) do
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
describe RequestLogAnalyzer::Aggregator::Summarizer, :combined_requests do
|
4
|
+
describe RequestLogAnalyzer::Aggregator::Summarizer do
|
11
5
|
|
12
6
|
before(:each) do
|
13
7
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wvanbergen-request-log-analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Willem van Bergen
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2009-01-
|
13
|
+
date: 2009-01-13 00:00:00 -08:00
|
14
14
|
default_executable: request-log-analyzer
|
15
15
|
dependencies: []
|
16
16
|
|