request-log-analyzer 1.3.7 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +2 -2
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer.rb +4 -4
- data/lib/request_log_analyzer/aggregator.rb +10 -10
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/controller.rb +153 -69
- data/lib/request_log_analyzer/database.rb +13 -13
- data/lib/request_log_analyzer/database/base.rb +17 -17
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +2 -2
- data/lib/request_log_analyzer/database/source.rb +1 -1
- data/lib/request_log_analyzer/file_format.rb +15 -15
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
- data/lib/request_log_analyzer/file_format/apache.rb +20 -19
- data/lib/request_log_analyzer/file_format/merb.rb +12 -12
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +146 -70
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/filter.rb +6 -6
- data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +22 -22
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/request.rb +35 -36
- data/lib/request_log_analyzer/source.rb +7 -7
- data/lib/request_log_analyzer/source/database_loader.rb +7 -7
- data/lib/request_log_analyzer/source/log_parser.rb +48 -43
- data/lib/request_log_analyzer/tracker.rb +128 -14
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/request-log-analyzer.gemspec +19 -15
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +1 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +19 -19
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +25 -25
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +13 -13
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +8 -6
data/lib/cli/tools.rb
CHANGED
@@ -4,15 +4,15 @@
|
|
4
4
|
def terminal_width(default_width = 81)
|
5
5
|
tiocgwinsz = 0x5413
|
6
6
|
data = [0, 0, 0, 0].pack("SSSS")
|
7
|
-
if @out.ioctl(tiocgwinsz, data) >= 0
|
7
|
+
if @out.ioctl(tiocgwinsz, data) >= 0
|
8
8
|
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
9
9
|
raise unless cols > 0
|
10
10
|
cols
|
11
11
|
else
|
12
12
|
raise
|
13
13
|
end
|
14
|
-
rescue
|
15
|
-
begin
|
14
|
+
rescue
|
15
|
+
begin
|
16
16
|
IO.popen('stty -a 2>&1') do |pipe|
|
17
17
|
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
18
18
|
raise unless column_line
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -8,10 +8,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
|
|
8
8
|
# - This module itselfs contains some functions to help with class and source file loading.
|
9
9
|
# - The actual application resides in the RequestLogAnalyzer::Controller class.
|
10
10
|
module RequestLogAnalyzer
|
11
|
-
|
11
|
+
|
12
12
|
# The current version of request-log-analyzer.
|
13
|
-
#
|
14
|
-
VERSION = "1.
|
13
|
+
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
14
|
+
VERSION = "1.4.0"
|
15
15
|
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
@@ -35,7 +35,7 @@ module RequestLogAnalyzer
|
|
35
35
|
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
36
36
|
end
|
37
37
|
|
38
|
-
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
38
|
+
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
39
39
|
# (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
|
40
40
|
# <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
|
41
41
|
def self.to_camelcase(str)
|
@@ -1,13 +1,13 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
2
|
+
|
3
3
|
def self.const_missing(const)
|
4
4
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
5
5
|
end
|
6
|
-
|
6
|
+
|
7
7
|
# The base class of an aggregator. This class provides the interface to which
|
8
8
|
# every aggregator should comply (by simply subclassing this class).
|
9
9
|
class Base
|
10
|
-
|
10
|
+
|
11
11
|
attr_reader :options, :source
|
12
12
|
|
13
13
|
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
@@ -17,30 +17,30 @@ module RequestLogAnalyzer::Aggregator
|
|
17
17
|
@options = options
|
18
18
|
end
|
19
19
|
|
20
|
-
# The prepare function is called just before parsing starts. This function
|
20
|
+
# The prepare function is called just before parsing starts. This function
|
21
21
|
# can be used to initialie variables, etc.
|
22
22
|
def prepare
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
# The aggregate function is called for every request.
|
26
26
|
# Implement the aggregating functionality in this method
|
27
27
|
def aggregate(request)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# The finalize function is called after all sources are parsed and no more
|
31
31
|
# requests will be passed to the aggregator
|
32
32
|
def finalize
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
# The warning method is called if the parser eits a warning.
|
36
36
|
def warning(type, message, lineno)
|
37
|
-
end
|
38
|
-
|
37
|
+
end
|
38
|
+
|
39
39
|
# The report function is called at the end. Implement any result reporting
|
40
40
|
# in this function.
|
41
41
|
def report(output)
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
# The source_change function gets called when handling a source is started or finished.
|
45
45
|
def source_change(change, filename)
|
46
46
|
end
|
@@ -4,7 +4,7 @@ module RequestLogAnalyzer::Aggregator
|
|
4
4
|
# The database aggregator will create an SQLite3 database with all parsed request information.
|
5
5
|
#
|
6
6
|
# The prepare method will create a database schema according to the file format definitions.
|
7
|
-
# It will also create ActiveRecord::Base subclasses to interact with the created tables.
|
7
|
+
# It will also create ActiveRecord::Base subclasses to interact with the created tables.
|
8
8
|
# Then, the aggregate method will be called for every parsed request. The information of
|
9
9
|
# these requests is inserted into the tables using the ActiveRecord classes.
|
10
10
|
#
|
@@ -22,11 +22,11 @@ module RequestLogAnalyzer::Aggregator
|
|
22
22
|
@sources = {}
|
23
23
|
@database = RequestLogAnalyzer::Database.new(options[:database])
|
24
24
|
@database.file_format = source.file_format
|
25
|
-
|
25
|
+
|
26
26
|
database.drop_database_schema! if options[:reset_database]
|
27
27
|
database.create_database_schema!
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# Aggregates a request into the database
|
31
31
|
# This will create a record in the requests table and create a record for every line that has been parsed,
|
32
32
|
# in which the captured values will be stored.
|
@@ -42,19 +42,19 @@ module RequestLogAnalyzer::Aggregator
|
|
42
42
|
rescue SQLite3::SQLException => e
|
43
43
|
raise Interrupt, e.message
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
# Finalizes the aggregator by closing the connection to the database
|
47
47
|
def finalize
|
48
48
|
@request_count = RequestLogAnalyzer::Database::Request.count
|
49
49
|
database.disconnect
|
50
50
|
database.remove_orm_classes!
|
51
51
|
end
|
52
|
-
|
52
|
+
|
53
53
|
# Records w warining in the warnings table.
|
54
54
|
def warning(type, message, lineno)
|
55
55
|
RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
|
56
56
|
end
|
57
|
-
|
57
|
+
|
58
58
|
# Records source changes in the sources table
|
59
59
|
def source_change(change, filename)
|
60
60
|
if File.exist?(filename)
|
@@ -66,11 +66,11 @@ module RequestLogAnalyzer::Aggregator
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
# Prints a short report of what has been inserted into the database
|
71
71
|
def report(output)
|
72
72
|
output.title('Request database created')
|
73
|
-
|
73
|
+
|
74
74
|
output << "A database file has been created with all parsed request information.\n"
|
75
75
|
output << "#{@request_count} requests have been added to the database.\n"
|
76
76
|
output << "\n"
|
@@ -78,6 +78,6 @@ module RequestLogAnalyzer::Aggregator
|
|
78
78
|
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
|
79
79
|
output << "\n"
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
end
|
83
83
|
end
|
@@ -1,23 +1,28 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
2
|
|
3
|
-
# Echo Aggregator. Writes everything passed to
|
3
|
+
# Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
|
4
4
|
class Echo < Base
|
5
|
-
|
5
|
+
|
6
|
+
attr_accessor :warnings
|
7
|
+
|
6
8
|
def prepare
|
7
|
-
@warnings =
|
9
|
+
@warnings = []
|
8
10
|
end
|
9
|
-
|
11
|
+
|
12
|
+
# Display every parsed line immediately to the terminal
|
10
13
|
def aggregate(request)
|
11
|
-
puts "\nRequest: " + request.inspect
|
14
|
+
puts "\nRequest: " + request.lines.inspect
|
12
15
|
end
|
13
|
-
|
16
|
+
|
17
|
+
# Capture all warnings during parsing
|
14
18
|
def warning(type, message, lineno)
|
15
|
-
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}
|
19
|
+
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}"
|
16
20
|
end
|
17
|
-
|
21
|
+
|
22
|
+
# Display every warning in the report when finished parsing
|
18
23
|
def report(output)
|
19
24
|
output.title("Warnings during parsing")
|
20
|
-
output.puts
|
25
|
+
@warnings.each { |w| output.puts(w) }
|
21
26
|
end
|
22
27
|
|
23
28
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
2
|
|
3
3
|
class Summarizer < Base
|
4
|
-
|
4
|
+
|
5
5
|
class Definer
|
6
|
-
|
6
|
+
|
7
7
|
attr_reader :trackers
|
8
8
|
|
9
9
|
# Initialize tracker array
|
@@ -16,17 +16,17 @@ module RequestLogAnalyzer::Aggregator
|
|
16
16
|
def initialize_copy(other)
|
17
17
|
@trackers = other.trackers.dup
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
# Drop all trackers
|
21
21
|
def reset!
|
22
22
|
@trackers = []
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
# Include missing trackers through method missing.
|
26
26
|
def method_missing(tracker_method, *args)
|
27
27
|
track(tracker_method, *args)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# Track the frequency of a specific category
|
31
31
|
# <tt>category_field</tt> Field to track
|
32
32
|
# <tt>options</tt> options are passed to new frequency tracker
|
@@ -37,7 +37,7 @@ module RequestLogAnalyzer::Aggregator
|
|
37
37
|
track(:frequency, category_field.merge(options))
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
# Track the duration of a specific category
|
42
42
|
# <tt>duration_field</tt> Field to track
|
43
43
|
# <tt>options</tt> options are passed to new frequency tracker
|
@@ -45,10 +45,10 @@ module RequestLogAnalyzer::Aggregator
|
|
45
45
|
if duration_field.kind_of?(Symbol)
|
46
46
|
track(:duration, options.merge(:duration => duration_field))
|
47
47
|
elsif duration_field.kind_of?(Hash)
|
48
|
-
track(:duration, duration_field.merge(options))
|
48
|
+
track(:duration, duration_field.merge(options))
|
49
49
|
end
|
50
|
-
end
|
51
|
-
|
50
|
+
end
|
51
|
+
|
52
52
|
# Helper function to initialize a tracker and add it to the tracker array.
|
53
53
|
# <tt>tracker_class</tt> The class to include
|
54
54
|
# <tt>optiont</tt> The options to pass to the trackers.
|
@@ -57,10 +57,10 @@ module RequestLogAnalyzer::Aggregator
|
|
57
57
|
@trackers << tracker_klass.new(options)
|
58
58
|
end
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
attr_reader :trackers
|
62
62
|
attr_reader :warnings_encountered
|
63
|
-
|
63
|
+
|
64
64
|
# Initialize summarizer.
|
65
65
|
# Generate trackers from speciefied source.file_format.report_trackers and set them up
|
66
66
|
def initialize(source, options = {})
|
@@ -69,16 +69,16 @@ module RequestLogAnalyzer::Aggregator
|
|
69
69
|
@trackers = source.file_format.report_trackers
|
70
70
|
setup
|
71
71
|
end
|
72
|
-
|
72
|
+
|
73
73
|
def setup
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
# Call prepare on all trackers.
|
77
77
|
def prepare
|
78
78
|
raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
|
79
79
|
@trackers.each { |tracker| tracker.prepare }
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
# Pass all requests to trackers and let them update if necessary.
|
83
83
|
# <tt>request</tt> The request to pass.
|
84
84
|
def aggregate(request)
|
@@ -86,19 +86,19 @@ module RequestLogAnalyzer::Aggregator
|
|
86
86
|
tracker.update(request) if tracker.should_update?(request)
|
87
87
|
end
|
88
88
|
end
|
89
|
-
|
89
|
+
|
90
90
|
# Call finalize on all trackers. Saves a YAML dump if this is set in the options.
|
91
91
|
def finalize
|
92
92
|
@trackers.each { |tracker| tracker.finalize }
|
93
93
|
save_results_dump(options[:dump]) if options[:dump]
|
94
94
|
end
|
95
|
-
|
95
|
+
|
96
96
|
# Saves the results of all the trackers in YAML format to a file.
|
97
97
|
# <tt>filename</tt> The file to store the YAML dump in.
|
98
98
|
def save_results_dump(filename)
|
99
99
|
File.open(filename, 'w') { |file| file.write(to_yaml) }
|
100
100
|
end
|
101
|
-
|
101
|
+
|
102
102
|
# Exports all the tracker results to YAML. It will call the to_yaml_object method
|
103
103
|
# for every tracker and combines these into a single YAML export.
|
104
104
|
def to_yaml
|
@@ -108,7 +108,7 @@ module RequestLogAnalyzer::Aggregator
|
|
108
108
|
end
|
109
109
|
YAML::dump(trackers_export)
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
# Call report on all trackers.
|
113
113
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
114
114
|
def report(output)
|
@@ -121,13 +121,13 @@ module RequestLogAnalyzer::Aggregator
|
|
121
121
|
end
|
122
122
|
report_footer(output)
|
123
123
|
end
|
124
|
-
|
124
|
+
|
125
125
|
# Generate report header.
|
126
126
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
127
127
|
def report_header(output)
|
128
128
|
output.title("Request summary")
|
129
|
-
|
130
|
-
output.with_style(:cell_separator => false) do
|
129
|
+
|
130
|
+
output.with_style(:cell_separator => false) do
|
131
131
|
output.table({:width => 20}, {:font => :bold}) do |rows|
|
132
132
|
rows << ['Parsed lines:', source.parsed_lines]
|
133
133
|
rows << ['Skipped lines:', source.skipped_lines]
|
@@ -138,13 +138,13 @@ module RequestLogAnalyzer::Aggregator
|
|
138
138
|
end
|
139
139
|
output << "\n"
|
140
140
|
end
|
141
|
-
|
141
|
+
|
142
142
|
# Generate report footer.
|
143
143
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
144
144
|
def report_footer(output)
|
145
145
|
if has_log_ordering_warnings?
|
146
146
|
output.title("Parse warnings")
|
147
|
-
|
147
|
+
|
148
148
|
output.puts "Parseable lines were ancountered without a header line before it. It"
|
149
149
|
output.puts "could be that logging is not setup correctly for your application."
|
150
150
|
output.puts "Visit this website for logging configuration tips:"
|
@@ -152,17 +152,17 @@ module RequestLogAnalyzer::Aggregator
|
|
152
152
|
output.puts
|
153
153
|
end
|
154
154
|
end
|
155
|
-
|
155
|
+
|
156
156
|
# Returns true if there were any warnings generated by the trackers
|
157
157
|
def has_warnings?
|
158
158
|
@warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
|
159
159
|
end
|
160
|
-
|
160
|
+
|
161
161
|
# Returns true if there were any log ordering warnings
|
162
162
|
def has_log_ordering_warnings?
|
163
163
|
@warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
|
164
164
|
end
|
165
|
-
|
165
|
+
|
166
166
|
# Store an encountered warning
|
167
167
|
# <tt>type</tt> Type of warning
|
168
168
|
# <tt>message</tt> Warning message
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer
|
2
|
-
|
2
|
+
|
3
3
|
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
4
4
|
# requested file format, and connect it with sources and aggregators.
|
5
5
|
#
|
@@ -22,37 +22,39 @@ module RequestLogAnalyzer
|
|
22
22
|
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
23
23
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
24
24
|
# <rr>report_with</tt> Width of the report. Defaults to 80.
|
25
|
-
def self.
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
def self.build_from_arguments(arguments)
|
26
|
+
|
27
|
+
options = {}
|
28
|
+
|
29
|
+
# Copy fields
|
30
|
+
options[:database] = arguments[:database]
|
30
31
|
options[:reset_database] = arguments[:reset_database]
|
31
32
|
options[:debug] = arguments[:debug]
|
32
33
|
options[:dump] = arguments[:dump]
|
33
34
|
options[:parse_strategy] = arguments[:parse_strategy]
|
34
35
|
options[:no_progress] = arguments[:no_progress]
|
36
|
+
options[:format] = arguments[:format]
|
37
|
+
options[:output] = arguments[:output]
|
38
|
+
options[:file] = arguments[:file]
|
39
|
+
options[:format] = arguments[:format]
|
40
|
+
options[:after] = arguments[:after]
|
41
|
+
options[:before] = arguments[:before]
|
42
|
+
options[:reject] = arguments[:reject]
|
43
|
+
options[:select] = arguments[:select]
|
44
|
+
options[:boring] = arguments[:boring]
|
45
|
+
options[:aggregator] = arguments[:aggregator]
|
46
|
+
options[:report_width] = arguments[:report_width]
|
47
|
+
options[:report_sort] = arguments[:report_sort]
|
48
|
+
options[:report_amount] = arguments[:report_amount]
|
35
49
|
|
36
|
-
|
37
|
-
if arguments[:
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
output_mail = RequestLogAnalyzer::Mailer.new(arguments[:mail])
|
42
|
-
options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)
|
43
|
-
else
|
44
|
-
options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i,
|
45
|
-
:color => !arguments[:boring], :characters => (arguments[:boring] ? :ascii : :utf))
|
50
|
+
# Apache format workaround
|
51
|
+
if arguments[:rails_format]
|
52
|
+
options[:format] = {:rails => arguments[:rails_format]}
|
53
|
+
elsif arguments[:apache_format]
|
54
|
+
options[:format] = {:apache => arguments[:apache_format]}
|
46
55
|
end
|
47
|
-
|
48
|
-
#
|
49
|
-
file_format = if arguments[:apache_format]
|
50
|
-
RequestLogAnalyzer::FileFormat.load(:apache, arguments[:apache_format])
|
51
|
-
else
|
52
|
-
RequestLogAnalyzer::FileFormat.load(arguments[:format])
|
53
|
-
end
|
54
|
-
|
55
|
-
# register sources
|
56
|
+
|
57
|
+
# Register sources
|
56
58
|
if arguments.parameters.length == 1
|
57
59
|
file = arguments.parameters[0]
|
58
60
|
if file == '-' || file == 'STDIN'
|
@@ -67,49 +69,128 @@ module RequestLogAnalyzer
|
|
67
69
|
options.store(:source_files, arguments.parameters)
|
68
70
|
end
|
69
71
|
|
70
|
-
|
71
|
-
|
72
|
+
build(options)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Build a new controller using parameters (Base for new API)
|
76
|
+
# <tt>source</tt> The source file
|
77
|
+
# Options are passd on to the LogParser.
|
78
|
+
#
|
79
|
+
# Options
|
80
|
+
# * <tt>:database</tt> Database file
|
81
|
+
# * <tt>:reset_database</tt>
|
82
|
+
# * <tt>:debug</tt> Enables echo aggregator.
|
83
|
+
# * <tt>:dump</tt>
|
84
|
+
# * <tt>:parse_strategy</tt>
|
85
|
+
# * <tt>:no_progress</tt>
|
86
|
+
# * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
|
87
|
+
# * <tt>:file</tt> Filestring or File or StringIO
|
88
|
+
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. Defaults to :rails.
|
89
|
+
# * <tt>:source_files</tt> File or STDIN
|
90
|
+
# * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
91
|
+
# * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
92
|
+
# * <tt>:reject</tt> Reject specific {:field => :value} combination. Expects single hash.
|
93
|
+
# * <tt>:select</tt> Select specific {:field => :value} combination. Expects single hash.
|
94
|
+
# * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY!). Defaults to [:summarizer
|
95
|
+
# * <tt>:boring</tt> Do not show color on STDOUT. Defaults to False.
|
96
|
+
# * <tt>:report_width</tt> Width or reports in characters. Defaults to 80.
|
97
|
+
#
|
98
|
+
# TODO:
|
99
|
+
# Check if defaults work (Aggregator defaults seem wrong).
|
100
|
+
# Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
|
101
|
+
def self.build(options)
|
102
|
+
# Defaults
|
103
|
+
options[:output] ||= :fixed_width
|
104
|
+
options[:format] ||= :rails
|
105
|
+
options[:aggregator] ||= [:summarizer]
|
106
|
+
options[:report_width] ||= 80
|
107
|
+
options[:report_amount] ||= 20
|
108
|
+
options[:report_sort] ||= 'sum,mean'
|
109
|
+
options[:boring] ||= false
|
110
|
+
|
111
|
+
# Set the output class
|
112
|
+
output_args = {}
|
113
|
+
output_object = nil
|
114
|
+
if options[:output].is_a? Class
|
115
|
+
output_class = options[:output]
|
116
|
+
else
|
117
|
+
output_class = RequestLogAnalyzer::Output::const_get(options[:output])
|
118
|
+
end
|
119
|
+
|
120
|
+
output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
|
121
|
+
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
122
|
+
|
123
|
+
if options[:file]
|
124
|
+
output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
|
125
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
126
|
+
elsif options[:mail]
|
127
|
+
output_object = RequestLogAnalyzer::Mailer.new(arguments[:mail])
|
128
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
129
|
+
else
|
130
|
+
output_object = STDOUT
|
131
|
+
output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
|
132
|
+
:characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
|
133
|
+
end
|
134
|
+
|
135
|
+
output_instance = output_class.new(output_object, output_args)
|
136
|
+
|
137
|
+
# Create the controller with the correct file format
|
138
|
+
if options[:format].kind_of?(Hash)
|
139
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
140
|
+
else
|
141
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
142
|
+
end
|
72
143
|
|
144
|
+
# Kickstart the controller
|
145
|
+
controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
|
146
|
+
{ :output => output_instance,
|
147
|
+
:database => options[:database], # FUGLY!
|
148
|
+
:dump => options[:dump],
|
149
|
+
:reset_database => options[:reset_database]})
|
150
|
+
|
73
151
|
# register filters
|
74
|
-
if
|
152
|
+
if options[:after] || options[:before]
|
75
153
|
filter_options = {}
|
76
|
-
|
77
|
-
|
154
|
+
[:after, :before].each do |filter|
|
155
|
+
case options[filter]
|
156
|
+
when Date, DateTime, Time
|
157
|
+
filter_options[filter] = options[filter]
|
158
|
+
when String
|
159
|
+
filter_options[filter] = DateTime.parse(options[filter])
|
160
|
+
end
|
161
|
+
end
|
78
162
|
controller.add_filter(:timespan, filter_options)
|
79
163
|
end
|
80
|
-
|
81
|
-
|
82
|
-
|
164
|
+
|
165
|
+
if options[:reject]
|
166
|
+
options[:reject].each do |(field, value)|
|
167
|
+
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
|
168
|
+
end
|
83
169
|
end
|
84
|
-
|
85
|
-
|
86
|
-
|
170
|
+
|
171
|
+
if options[:reject]
|
172
|
+
options[:select].each do |(field, value)|
|
173
|
+
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
174
|
+
end
|
87
175
|
end
|
88
176
|
|
89
177
|
# register aggregators
|
90
|
-
|
178
|
+
options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
|
179
|
+
controller.add_aggregator(:summarizer) if options[:aggregator].empty?
|
180
|
+
controller.add_aggregator(:echo) if options[:debug]
|
181
|
+
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
91
182
|
|
92
|
-
# register the database
|
93
|
-
controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
|
94
|
-
controller.add_aggregator(:database_inserter) if arguments[:database] && !arguments[:aggregator].include?('database')
|
95
|
-
|
96
|
-
# register the echo aggregator in debug mode
|
97
|
-
controller.add_aggregator(:echo) if arguments[:debug]
|
98
|
-
|
99
183
|
file_format.setup_environment(controller)
|
100
|
-
|
101
184
|
return controller
|
102
|
-
end
|
185
|
+
end
|
103
186
|
|
104
187
|
# Builds a new Controller for the given log file format.
|
105
188
|
# <tt>format</tt> Logfile format. Defaults to :rails
|
106
189
|
# Options are passd on to the LogParser.
|
107
|
-
# * <tt>:aggregator</tt> Aggregator array.
|
108
190
|
# * <tt>:database</tt> Database the controller should use.
|
109
|
-
# * <tt>:
|
110
|
-
# * <tt>:silent</tt> Do not output any warnings.
|
111
|
-
# * <tt>:colorize</tt> Colorize output
|
191
|
+
# * <tt>:dump</tt> Yaml Dump the contrller should use.
|
112
192
|
# * <tt>:output</tt> All report outputs get << through this output.
|
193
|
+
# * <tt>:no_progress</tt> No progress bar
|
113
194
|
def initialize(source, options = {})
|
114
195
|
|
115
196
|
@source = source
|
@@ -117,16 +198,17 @@ module RequestLogAnalyzer
|
|
117
198
|
@aggregators = []
|
118
199
|
@filters = []
|
119
200
|
@output = options[:output]
|
201
|
+
@interrupted = false
|
120
202
|
|
121
203
|
# Register the request format for this session after checking its validity
|
122
204
|
raise "Invalid file format!" unless @source.file_format.valid?
|
123
|
-
|
205
|
+
|
124
206
|
# Install event handlers for wrnings, progress updates and source changes
|
125
207
|
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
126
208
|
@source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
|
127
209
|
@source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
|
128
210
|
end
|
129
|
-
|
211
|
+
|
130
212
|
# Progress function.
|
131
213
|
# Expects :started with file, :progress with current line and :finished or :interrupted when done.
|
132
214
|
# <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
|
@@ -147,46 +229,46 @@ module RequestLogAnalyzer
|
|
147
229
|
@progress_bar.set(value)
|
148
230
|
end
|
149
231
|
end
|
150
|
-
|
232
|
+
|
151
233
|
# Source change handler
|
152
234
|
def handle_source_change(change, filename)
|
153
235
|
@aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
|
154
236
|
end
|
155
|
-
|
156
|
-
# Adds an aggregator to the controller. The aggregator will be called for every request
|
237
|
+
|
238
|
+
# Adds an aggregator to the controller. The aggregator will be called for every request
|
157
239
|
# that is parsed from the provided sources (see add_source)
|
158
|
-
def add_aggregator(agg)
|
240
|
+
def add_aggregator(agg)
|
159
241
|
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
|
160
242
|
@aggregators << agg.new(@source, @options)
|
161
243
|
end
|
162
|
-
|
244
|
+
|
163
245
|
alias :>> :add_aggregator
|
164
|
-
|
246
|
+
|
165
247
|
# Adds a request filter to the controller.
|
166
248
|
def add_filter(filter, filter_options = {})
|
167
249
|
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
|
168
250
|
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
169
251
|
end
|
170
|
-
|
252
|
+
|
171
253
|
# Push a request through the entire filterchain (@filters).
|
172
254
|
# <tt>request</tt> The request to filter.
|
173
255
|
# Returns the filtered request or nil.
|
174
256
|
def filter_request(request)
|
175
|
-
@filters.each do |filter|
|
257
|
+
@filters.each do |filter|
|
176
258
|
request = filter.filter(request)
|
177
259
|
return nil if request.nil?
|
178
260
|
end
|
179
261
|
return request
|
180
262
|
end
|
181
|
-
|
263
|
+
|
182
264
|
# Push a request to all the aggregators (@aggregators).
|
183
|
-
# <tt>request</tt> The request to push to the aggregators.
|
265
|
+
# <tt>request</tt> The request to push to the aggregators.
|
184
266
|
def aggregate_request(request)
|
185
267
|
return false unless request
|
186
268
|
@aggregators.each { |agg| agg.aggregate(request) }
|
187
269
|
return true
|
188
270
|
end
|
189
|
-
|
271
|
+
|
190
272
|
# Runs RequestLogAnalyzer
|
191
273
|
# 1. Call prepare on every aggregator
|
192
274
|
# 2. Generate requests from source object
|
@@ -196,10 +278,12 @@ module RequestLogAnalyzer
|
|
196
278
|
# 5. Call report on every aggregator
|
197
279
|
# 6. Finalize Source
|
198
280
|
def run!
|
199
|
-
|
281
|
+
|
282
|
+
# @aggregators.each{|agg| p agg}
|
283
|
+
|
200
284
|
@aggregators.each { |agg| agg.prepare }
|
201
285
|
install_signal_handlers
|
202
|
-
|
286
|
+
|
203
287
|
@source.each_request do |request|
|
204
288
|
break if @interrupted
|
205
289
|
aggregate_request(filter_request(request))
|
@@ -210,9 +294,9 @@ module RequestLogAnalyzer
|
|
210
294
|
@output.header
|
211
295
|
@aggregators.each { |agg| agg.report(@output) }
|
212
296
|
@output.footer
|
213
|
-
|
297
|
+
|
214
298
|
@source.finalize
|
215
|
-
|
299
|
+
|
216
300
|
if @output.io.kind_of?(File)
|
217
301
|
puts
|
218
302
|
puts "Report written to: " + File.expand_path(@output.io.path)
|
@@ -224,7 +308,7 @@ module RequestLogAnalyzer
|
|
224
308
|
@output.io.mail
|
225
309
|
end
|
226
310
|
end
|
227
|
-
|
311
|
+
|
228
312
|
def install_signal_handlers
|
229
313
|
Signal.trap("INT") do
|
230
314
|
handle_progress(:interrupted)
|
@@ -232,6 +316,6 @@ module RequestLogAnalyzer
|
|
232
316
|
@interrupted = true
|
233
317
|
end
|
234
318
|
end
|
235
|
-
|
319
|
+
|
236
320
|
end
|
237
321
|
end
|