request-log-analyzer 1.3.7 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +2 -2
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer.rb +4 -4
- data/lib/request_log_analyzer/aggregator.rb +10 -10
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/controller.rb +153 -69
- data/lib/request_log_analyzer/database.rb +13 -13
- data/lib/request_log_analyzer/database/base.rb +17 -17
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +2 -2
- data/lib/request_log_analyzer/database/source.rb +1 -1
- data/lib/request_log_analyzer/file_format.rb +15 -15
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
- data/lib/request_log_analyzer/file_format/apache.rb +20 -19
- data/lib/request_log_analyzer/file_format/merb.rb +12 -12
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +146 -70
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/filter.rb +6 -6
- data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +22 -22
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/request.rb +35 -36
- data/lib/request_log_analyzer/source.rb +7 -7
- data/lib/request_log_analyzer/source/database_loader.rb +7 -7
- data/lib/request_log_analyzer/source/log_parser.rb +48 -43
- data/lib/request_log_analyzer/tracker.rb +128 -14
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/request-log-analyzer.gemspec +19 -15
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +1 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +19 -19
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +25 -25
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +13 -13
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +8 -6
data/lib/cli/tools.rb
CHANGED
@@ -4,15 +4,15 @@
|
|
4
4
|
def terminal_width(default_width = 81)
|
5
5
|
tiocgwinsz = 0x5413
|
6
6
|
data = [0, 0, 0, 0].pack("SSSS")
|
7
|
-
if @out.ioctl(tiocgwinsz, data) >= 0
|
7
|
+
if @out.ioctl(tiocgwinsz, data) >= 0
|
8
8
|
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
9
9
|
raise unless cols > 0
|
10
10
|
cols
|
11
11
|
else
|
12
12
|
raise
|
13
13
|
end
|
14
|
-
rescue
|
15
|
-
begin
|
14
|
+
rescue
|
15
|
+
begin
|
16
16
|
IO.popen('stty -a 2>&1') do |pipe|
|
17
17
|
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
18
18
|
raise unless column_line
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -8,10 +8,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
|
|
8
8
|
# - This module itselfs contains some functions to help with class and source file loading.
|
9
9
|
# - The actual application resides in the RequestLogAnalyzer::Controller class.
|
10
10
|
module RequestLogAnalyzer
|
11
|
-
|
11
|
+
|
12
12
|
# The current version of request-log-analyzer.
|
13
|
-
#
|
14
|
-
VERSION = "1.
|
13
|
+
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
14
|
+
VERSION = "1.4.0"
|
15
15
|
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
@@ -35,7 +35,7 @@ module RequestLogAnalyzer
|
|
35
35
|
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
36
36
|
end
|
37
37
|
|
38
|
-
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
38
|
+
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
39
39
|
# (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
|
40
40
|
# <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
|
41
41
|
def self.to_camelcase(str)
|
@@ -1,13 +1,13 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
|
-
|
2
|
+
|
3
3
|
def self.const_missing(const)
|
4
4
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
5
5
|
end
|
6
|
-
|
6
|
+
|
7
7
|
# The base class of an aggregator. This class provides the interface to which
|
8
8
|
# every aggregator should comply (by simply subclassing this class).
|
9
9
|
class Base
|
10
|
-
|
10
|
+
|
11
11
|
attr_reader :options, :source
|
12
12
|
|
13
13
|
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
@@ -17,30 +17,30 @@ module RequestLogAnalyzer::Aggregator
|
|
17
17
|
@options = options
|
18
18
|
end
|
19
19
|
|
20
|
-
# The prepare function is called just before parsing starts. This function
|
20
|
+
# The prepare function is called just before parsing starts. This function
|
21
21
|
# can be used to initialie variables, etc.
|
22
22
|
def prepare
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
# The aggregate function is called for every request.
|
26
26
|
# Implement the aggregating functionality in this method
|
27
27
|
def aggregate(request)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# The finalize function is called after all sources are parsed and no more
|
31
31
|
# requests will be passed to the aggregator
|
32
32
|
def finalize
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
# The warning method is called if the parser eits a warning.
|
36
36
|
def warning(type, message, lineno)
|
37
|
-
end
|
38
|
-
|
37
|
+
end
|
38
|
+
|
39
39
|
# The report function is called at the end. Implement any result reporting
|
40
40
|
# in this function.
|
41
41
|
def report(output)
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
# The source_change function gets called when handling a source is started or finished.
|
45
45
|
def source_change(change, filename)
|
46
46
|
end
|
@@ -4,7 +4,7 @@ module RequestLogAnalyzer::Aggregator
|
|
4
4
|
# The database aggregator will create an SQLite3 database with all parsed request information.
|
5
5
|
#
|
6
6
|
# The prepare method will create a database schema according to the file format definitions.
|
7
|
-
# It will also create ActiveRecord::Base subclasses to interact with the created tables.
|
7
|
+
# It will also create ActiveRecord::Base subclasses to interact with the created tables.
|
8
8
|
# Then, the aggregate method will be called for every parsed request. The information of
|
9
9
|
# these requests is inserted into the tables using the ActiveRecord classes.
|
10
10
|
#
|
@@ -22,11 +22,11 @@ module RequestLogAnalyzer::Aggregator
|
|
22
22
|
@sources = {}
|
23
23
|
@database = RequestLogAnalyzer::Database.new(options[:database])
|
24
24
|
@database.file_format = source.file_format
|
25
|
-
|
25
|
+
|
26
26
|
database.drop_database_schema! if options[:reset_database]
|
27
27
|
database.create_database_schema!
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# Aggregates a request into the database
|
31
31
|
# This will create a record in the requests table and create a record for every line that has been parsed,
|
32
32
|
# in which the captured values will be stored.
|
@@ -42,19 +42,19 @@ module RequestLogAnalyzer::Aggregator
|
|
42
42
|
rescue SQLite3::SQLException => e
|
43
43
|
raise Interrupt, e.message
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
# Finalizes the aggregator by closing the connection to the database
|
47
47
|
def finalize
|
48
48
|
@request_count = RequestLogAnalyzer::Database::Request.count
|
49
49
|
database.disconnect
|
50
50
|
database.remove_orm_classes!
|
51
51
|
end
|
52
|
-
|
52
|
+
|
53
53
|
# Records w warining in the warnings table.
|
54
54
|
def warning(type, message, lineno)
|
55
55
|
RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
|
56
56
|
end
|
57
|
-
|
57
|
+
|
58
58
|
# Records source changes in the sources table
|
59
59
|
def source_change(change, filename)
|
60
60
|
if File.exist?(filename)
|
@@ -66,11 +66,11 @@ module RequestLogAnalyzer::Aggregator
|
|
66
66
|
end
|
67
67
|
end
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
# Prints a short report of what has been inserted into the database
|
71
71
|
def report(output)
|
72
72
|
output.title('Request database created')
|
73
|
-
|
73
|
+
|
74
74
|
output << "A database file has been created with all parsed request information.\n"
|
75
75
|
output << "#{@request_count} requests have been added to the database.\n"
|
76
76
|
output << "\n"
|
@@ -78,6 +78,6 @@ module RequestLogAnalyzer::Aggregator
|
|
78
78
|
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
|
79
79
|
output << "\n"
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
end
|
83
83
|
end
|
@@ -1,23 +1,28 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
2
|
|
3
|
-
# Echo Aggregator. Writes everything passed to
|
3
|
+
# Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
|
4
4
|
class Echo < Base
|
5
|
-
|
5
|
+
|
6
|
+
attr_accessor :warnings
|
7
|
+
|
6
8
|
def prepare
|
7
|
-
@warnings =
|
9
|
+
@warnings = []
|
8
10
|
end
|
9
|
-
|
11
|
+
|
12
|
+
# Display every parsed line immediately to the terminal
|
10
13
|
def aggregate(request)
|
11
|
-
puts "\nRequest: " + request.inspect
|
14
|
+
puts "\nRequest: " + request.lines.inspect
|
12
15
|
end
|
13
|
-
|
16
|
+
|
17
|
+
# Capture all warnings during parsing
|
14
18
|
def warning(type, message, lineno)
|
15
|
-
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}
|
19
|
+
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}"
|
16
20
|
end
|
17
|
-
|
21
|
+
|
22
|
+
# Display every warning in the report when finished parsing
|
18
23
|
def report(output)
|
19
24
|
output.title("Warnings during parsing")
|
20
|
-
output.puts
|
25
|
+
@warnings.each { |w| output.puts(w) }
|
21
26
|
end
|
22
27
|
|
23
28
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
2
|
|
3
3
|
class Summarizer < Base
|
4
|
-
|
4
|
+
|
5
5
|
class Definer
|
6
|
-
|
6
|
+
|
7
7
|
attr_reader :trackers
|
8
8
|
|
9
9
|
# Initialize tracker array
|
@@ -16,17 +16,17 @@ module RequestLogAnalyzer::Aggregator
|
|
16
16
|
def initialize_copy(other)
|
17
17
|
@trackers = other.trackers.dup
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
# Drop all trackers
|
21
21
|
def reset!
|
22
22
|
@trackers = []
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
# Include missing trackers through method missing.
|
26
26
|
def method_missing(tracker_method, *args)
|
27
27
|
track(tracker_method, *args)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# Track the frequency of a specific category
|
31
31
|
# <tt>category_field</tt> Field to track
|
32
32
|
# <tt>options</tt> options are passed to new frequency tracker
|
@@ -37,7 +37,7 @@ module RequestLogAnalyzer::Aggregator
|
|
37
37
|
track(:frequency, category_field.merge(options))
|
38
38
|
end
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
# Track the duration of a specific category
|
42
42
|
# <tt>duration_field</tt> Field to track
|
43
43
|
# <tt>options</tt> options are passed to new frequency tracker
|
@@ -45,10 +45,10 @@ module RequestLogAnalyzer::Aggregator
|
|
45
45
|
if duration_field.kind_of?(Symbol)
|
46
46
|
track(:duration, options.merge(:duration => duration_field))
|
47
47
|
elsif duration_field.kind_of?(Hash)
|
48
|
-
track(:duration, duration_field.merge(options))
|
48
|
+
track(:duration, duration_field.merge(options))
|
49
49
|
end
|
50
|
-
end
|
51
|
-
|
50
|
+
end
|
51
|
+
|
52
52
|
# Helper function to initialize a tracker and add it to the tracker array.
|
53
53
|
# <tt>tracker_class</tt> The class to include
|
54
54
|
# <tt>optiont</tt> The options to pass to the trackers.
|
@@ -57,10 +57,10 @@ module RequestLogAnalyzer::Aggregator
|
|
57
57
|
@trackers << tracker_klass.new(options)
|
58
58
|
end
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
attr_reader :trackers
|
62
62
|
attr_reader :warnings_encountered
|
63
|
-
|
63
|
+
|
64
64
|
# Initialize summarizer.
|
65
65
|
# Generate trackers from speciefied source.file_format.report_trackers and set them up
|
66
66
|
def initialize(source, options = {})
|
@@ -69,16 +69,16 @@ module RequestLogAnalyzer::Aggregator
|
|
69
69
|
@trackers = source.file_format.report_trackers
|
70
70
|
setup
|
71
71
|
end
|
72
|
-
|
72
|
+
|
73
73
|
def setup
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
# Call prepare on all trackers.
|
77
77
|
def prepare
|
78
78
|
raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
|
79
79
|
@trackers.each { |tracker| tracker.prepare }
|
80
80
|
end
|
81
|
-
|
81
|
+
|
82
82
|
# Pass all requests to trackers and let them update if necessary.
|
83
83
|
# <tt>request</tt> The request to pass.
|
84
84
|
def aggregate(request)
|
@@ -86,19 +86,19 @@ module RequestLogAnalyzer::Aggregator
|
|
86
86
|
tracker.update(request) if tracker.should_update?(request)
|
87
87
|
end
|
88
88
|
end
|
89
|
-
|
89
|
+
|
90
90
|
# Call finalize on all trackers. Saves a YAML dump if this is set in the options.
|
91
91
|
def finalize
|
92
92
|
@trackers.each { |tracker| tracker.finalize }
|
93
93
|
save_results_dump(options[:dump]) if options[:dump]
|
94
94
|
end
|
95
|
-
|
95
|
+
|
96
96
|
# Saves the results of all the trackers in YAML format to a file.
|
97
97
|
# <tt>filename</tt> The file to store the YAML dump in.
|
98
98
|
def save_results_dump(filename)
|
99
99
|
File.open(filename, 'w') { |file| file.write(to_yaml) }
|
100
100
|
end
|
101
|
-
|
101
|
+
|
102
102
|
# Exports all the tracker results to YAML. It will call the to_yaml_object method
|
103
103
|
# for every tracker and combines these into a single YAML export.
|
104
104
|
def to_yaml
|
@@ -108,7 +108,7 @@ module RequestLogAnalyzer::Aggregator
|
|
108
108
|
end
|
109
109
|
YAML::dump(trackers_export)
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
# Call report on all trackers.
|
113
113
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
114
114
|
def report(output)
|
@@ -121,13 +121,13 @@ module RequestLogAnalyzer::Aggregator
|
|
121
121
|
end
|
122
122
|
report_footer(output)
|
123
123
|
end
|
124
|
-
|
124
|
+
|
125
125
|
# Generate report header.
|
126
126
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
127
127
|
def report_header(output)
|
128
128
|
output.title("Request summary")
|
129
|
-
|
130
|
-
output.with_style(:cell_separator => false) do
|
129
|
+
|
130
|
+
output.with_style(:cell_separator => false) do
|
131
131
|
output.table({:width => 20}, {:font => :bold}) do |rows|
|
132
132
|
rows << ['Parsed lines:', source.parsed_lines]
|
133
133
|
rows << ['Skipped lines:', source.skipped_lines]
|
@@ -138,13 +138,13 @@ module RequestLogAnalyzer::Aggregator
|
|
138
138
|
end
|
139
139
|
output << "\n"
|
140
140
|
end
|
141
|
-
|
141
|
+
|
142
142
|
# Generate report footer.
|
143
143
|
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
144
144
|
def report_footer(output)
|
145
145
|
if has_log_ordering_warnings?
|
146
146
|
output.title("Parse warnings")
|
147
|
-
|
147
|
+
|
148
148
|
output.puts "Parseable lines were ancountered without a header line before it. It"
|
149
149
|
output.puts "could be that logging is not setup correctly for your application."
|
150
150
|
output.puts "Visit this website for logging configuration tips:"
|
@@ -152,17 +152,17 @@ module RequestLogAnalyzer::Aggregator
|
|
152
152
|
output.puts
|
153
153
|
end
|
154
154
|
end
|
155
|
-
|
155
|
+
|
156
156
|
# Returns true if there were any warnings generated by the trackers
|
157
157
|
def has_warnings?
|
158
158
|
@warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
|
159
159
|
end
|
160
|
-
|
160
|
+
|
161
161
|
# Returns true if there were any log ordering warnings
|
162
162
|
def has_log_ordering_warnings?
|
163
163
|
@warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
|
164
164
|
end
|
165
|
-
|
165
|
+
|
166
166
|
# Store an encountered warning
|
167
167
|
# <tt>type</tt> Type of warning
|
168
168
|
# <tt>message</tt> Warning message
|
@@ -1,5 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer
|
2
|
-
|
2
|
+
|
3
3
|
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
4
4
|
# requested file format, and connect it with sources and aggregators.
|
5
5
|
#
|
@@ -22,37 +22,39 @@ module RequestLogAnalyzer
|
|
22
22
|
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
23
23
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
24
24
|
# <rr>report_with</tt> Width of the report. Defaults to 80.
|
25
|
-
def self.
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
25
|
+
def self.build_from_arguments(arguments)
|
26
|
+
|
27
|
+
options = {}
|
28
|
+
|
29
|
+
# Copy fields
|
30
|
+
options[:database] = arguments[:database]
|
30
31
|
options[:reset_database] = arguments[:reset_database]
|
31
32
|
options[:debug] = arguments[:debug]
|
32
33
|
options[:dump] = arguments[:dump]
|
33
34
|
options[:parse_strategy] = arguments[:parse_strategy]
|
34
35
|
options[:no_progress] = arguments[:no_progress]
|
36
|
+
options[:format] = arguments[:format]
|
37
|
+
options[:output] = arguments[:output]
|
38
|
+
options[:file] = arguments[:file]
|
39
|
+
options[:format] = arguments[:format]
|
40
|
+
options[:after] = arguments[:after]
|
41
|
+
options[:before] = arguments[:before]
|
42
|
+
options[:reject] = arguments[:reject]
|
43
|
+
options[:select] = arguments[:select]
|
44
|
+
options[:boring] = arguments[:boring]
|
45
|
+
options[:aggregator] = arguments[:aggregator]
|
46
|
+
options[:report_width] = arguments[:report_width]
|
47
|
+
options[:report_sort] = arguments[:report_sort]
|
48
|
+
options[:report_amount] = arguments[:report_amount]
|
35
49
|
|
36
|
-
|
37
|
-
if arguments[:
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
output_mail = RequestLogAnalyzer::Mailer.new(arguments[:mail])
|
42
|
-
options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)
|
43
|
-
else
|
44
|
-
options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i,
|
45
|
-
:color => !arguments[:boring], :characters => (arguments[:boring] ? :ascii : :utf))
|
50
|
+
# Apache format workaround
|
51
|
+
if arguments[:rails_format]
|
52
|
+
options[:format] = {:rails => arguments[:rails_format]}
|
53
|
+
elsif arguments[:apache_format]
|
54
|
+
options[:format] = {:apache => arguments[:apache_format]}
|
46
55
|
end
|
47
|
-
|
48
|
-
#
|
49
|
-
file_format = if arguments[:apache_format]
|
50
|
-
RequestLogAnalyzer::FileFormat.load(:apache, arguments[:apache_format])
|
51
|
-
else
|
52
|
-
RequestLogAnalyzer::FileFormat.load(arguments[:format])
|
53
|
-
end
|
54
|
-
|
55
|
-
# register sources
|
56
|
+
|
57
|
+
# Register sources
|
56
58
|
if arguments.parameters.length == 1
|
57
59
|
file = arguments.parameters[0]
|
58
60
|
if file == '-' || file == 'STDIN'
|
@@ -67,49 +69,128 @@ module RequestLogAnalyzer
|
|
67
69
|
options.store(:source_files, arguments.parameters)
|
68
70
|
end
|
69
71
|
|
70
|
-
|
71
|
-
|
72
|
+
build(options)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Build a new controller using parameters (Base for new API)
|
76
|
+
# <tt>source</tt> The source file
|
77
|
+
# Options are passd on to the LogParser.
|
78
|
+
#
|
79
|
+
# Options
|
80
|
+
# * <tt>:database</tt> Database file
|
81
|
+
# * <tt>:reset_database</tt>
|
82
|
+
# * <tt>:debug</tt> Enables echo aggregator.
|
83
|
+
# * <tt>:dump</tt>
|
84
|
+
# * <tt>:parse_strategy</tt>
|
85
|
+
# * <tt>:no_progress</tt>
|
86
|
+
# * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
|
87
|
+
# * <tt>:file</tt> Filestring or File or StringIO
|
88
|
+
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. Defaults to :rails.
|
89
|
+
# * <tt>:source_files</tt> File or STDIN
|
90
|
+
# * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
91
|
+
# * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
92
|
+
# * <tt>:reject</tt> Reject specific {:field => :value} combination. Expects single hash.
|
93
|
+
# * <tt>:select</tt> Select specific {:field => :value} combination. Expects single hash.
|
94
|
+
# * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY!). Defaults to [:summarizer
|
95
|
+
# * <tt>:boring</tt> Do not show color on STDOUT. Defaults to False.
|
96
|
+
# * <tt>:report_width</tt> Width or reports in characters. Defaults to 80.
|
97
|
+
#
|
98
|
+
# TODO:
|
99
|
+
# Check if defaults work (Aggregator defaults seem wrong).
|
100
|
+
# Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
|
101
|
+
def self.build(options)
|
102
|
+
# Defaults
|
103
|
+
options[:output] ||= :fixed_width
|
104
|
+
options[:format] ||= :rails
|
105
|
+
options[:aggregator] ||= [:summarizer]
|
106
|
+
options[:report_width] ||= 80
|
107
|
+
options[:report_amount] ||= 20
|
108
|
+
options[:report_sort] ||= 'sum,mean'
|
109
|
+
options[:boring] ||= false
|
110
|
+
|
111
|
+
# Set the output class
|
112
|
+
output_args = {}
|
113
|
+
output_object = nil
|
114
|
+
if options[:output].is_a? Class
|
115
|
+
output_class = options[:output]
|
116
|
+
else
|
117
|
+
output_class = RequestLogAnalyzer::Output::const_get(options[:output])
|
118
|
+
end
|
119
|
+
|
120
|
+
output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
|
121
|
+
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
122
|
+
|
123
|
+
if options[:file]
|
124
|
+
output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
|
125
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
126
|
+
elsif options[:mail]
|
127
|
+
output_object = RequestLogAnalyzer::Mailer.new(arguments[:mail])
|
128
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
129
|
+
else
|
130
|
+
output_object = STDOUT
|
131
|
+
output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
|
132
|
+
:characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
|
133
|
+
end
|
134
|
+
|
135
|
+
output_instance = output_class.new(output_object, output_args)
|
136
|
+
|
137
|
+
# Create the controller with the correct file format
|
138
|
+
if options[:format].kind_of?(Hash)
|
139
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
140
|
+
else
|
141
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
142
|
+
end
|
72
143
|
|
144
|
+
# Kickstart the controller
|
145
|
+
controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
|
146
|
+
{ :output => output_instance,
|
147
|
+
:database => options[:database], # FUGLY!
|
148
|
+
:dump => options[:dump],
|
149
|
+
:reset_database => options[:reset_database]})
|
150
|
+
|
73
151
|
# register filters
|
74
|
-
if
|
152
|
+
if options[:after] || options[:before]
|
75
153
|
filter_options = {}
|
76
|
-
|
77
|
-
|
154
|
+
[:after, :before].each do |filter|
|
155
|
+
case options[filter]
|
156
|
+
when Date, DateTime, Time
|
157
|
+
filter_options[filter] = options[filter]
|
158
|
+
when String
|
159
|
+
filter_options[filter] = DateTime.parse(options[filter])
|
160
|
+
end
|
161
|
+
end
|
78
162
|
controller.add_filter(:timespan, filter_options)
|
79
163
|
end
|
80
|
-
|
81
|
-
|
82
|
-
|
164
|
+
|
165
|
+
if options[:reject]
|
166
|
+
options[:reject].each do |(field, value)|
|
167
|
+
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
|
168
|
+
end
|
83
169
|
end
|
84
|
-
|
85
|
-
|
86
|
-
|
170
|
+
|
171
|
+
if options[:reject]
|
172
|
+
options[:select].each do |(field, value)|
|
173
|
+
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
174
|
+
end
|
87
175
|
end
|
88
176
|
|
89
177
|
# register aggregators
|
90
|
-
|
178
|
+
options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
|
179
|
+
controller.add_aggregator(:summarizer) if options[:aggregator].empty?
|
180
|
+
controller.add_aggregator(:echo) if options[:debug]
|
181
|
+
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
91
182
|
|
92
|
-
# register the database
|
93
|
-
controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
|
94
|
-
controller.add_aggregator(:database_inserter) if arguments[:database] && !arguments[:aggregator].include?('database')
|
95
|
-
|
96
|
-
# register the echo aggregator in debug mode
|
97
|
-
controller.add_aggregator(:echo) if arguments[:debug]
|
98
|
-
|
99
183
|
file_format.setup_environment(controller)
|
100
|
-
|
101
184
|
return controller
|
102
|
-
end
|
185
|
+
end
|
103
186
|
|
104
187
|
# Builds a new Controller for the given log file format.
|
105
188
|
# <tt>format</tt> Logfile format. Defaults to :rails
|
106
189
|
# Options are passd on to the LogParser.
|
107
|
-
# * <tt>:aggregator</tt> Aggregator array.
|
108
190
|
# * <tt>:database</tt> Database the controller should use.
|
109
|
-
# * <tt>:
|
110
|
-
# * <tt>:silent</tt> Do not output any warnings.
|
111
|
-
# * <tt>:colorize</tt> Colorize output
|
191
|
+
# * <tt>:dump</tt> Yaml Dump the contrller should use.
|
112
192
|
# * <tt>:output</tt> All report outputs get << through this output.
|
193
|
+
# * <tt>:no_progress</tt> No progress bar
|
113
194
|
def initialize(source, options = {})
|
114
195
|
|
115
196
|
@source = source
|
@@ -117,16 +198,17 @@ module RequestLogAnalyzer
|
|
117
198
|
@aggregators = []
|
118
199
|
@filters = []
|
119
200
|
@output = options[:output]
|
201
|
+
@interrupted = false
|
120
202
|
|
121
203
|
# Register the request format for this session after checking its validity
|
122
204
|
raise "Invalid file format!" unless @source.file_format.valid?
|
123
|
-
|
205
|
+
|
124
206
|
# Install event handlers for wrnings, progress updates and source changes
|
125
207
|
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
126
208
|
@source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
|
127
209
|
@source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
|
128
210
|
end
|
129
|
-
|
211
|
+
|
130
212
|
# Progress function.
|
131
213
|
# Expects :started with file, :progress with current line and :finished or :interrupted when done.
|
132
214
|
# <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
|
@@ -147,46 +229,46 @@ module RequestLogAnalyzer
|
|
147
229
|
@progress_bar.set(value)
|
148
230
|
end
|
149
231
|
end
|
150
|
-
|
232
|
+
|
151
233
|
# Source change handler
|
152
234
|
def handle_source_change(change, filename)
|
153
235
|
@aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
|
154
236
|
end
|
155
|
-
|
156
|
-
# Adds an aggregator to the controller. The aggregator will be called for every request
|
237
|
+
|
238
|
+
# Adds an aggregator to the controller. The aggregator will be called for every request
|
157
239
|
# that is parsed from the provided sources (see add_source)
|
158
|
-
def add_aggregator(agg)
|
240
|
+
def add_aggregator(agg)
|
159
241
|
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
|
160
242
|
@aggregators << agg.new(@source, @options)
|
161
243
|
end
|
162
|
-
|
244
|
+
|
163
245
|
alias :>> :add_aggregator
|
164
|
-
|
246
|
+
|
165
247
|
# Adds a request filter to the controller.
|
166
248
|
def add_filter(filter, filter_options = {})
|
167
249
|
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
|
168
250
|
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
169
251
|
end
|
170
|
-
|
252
|
+
|
171
253
|
# Push a request through the entire filterchain (@filters).
|
172
254
|
# <tt>request</tt> The request to filter.
|
173
255
|
# Returns the filtered request or nil.
|
174
256
|
def filter_request(request)
|
175
|
-
@filters.each do |filter|
|
257
|
+
@filters.each do |filter|
|
176
258
|
request = filter.filter(request)
|
177
259
|
return nil if request.nil?
|
178
260
|
end
|
179
261
|
return request
|
180
262
|
end
|
181
|
-
|
263
|
+
|
182
264
|
# Push a request to all the aggregators (@aggregators).
|
183
|
-
# <tt>request</tt> The request to push to the aggregators.
|
265
|
+
# <tt>request</tt> The request to push to the aggregators.
|
184
266
|
def aggregate_request(request)
|
185
267
|
return false unless request
|
186
268
|
@aggregators.each { |agg| agg.aggregate(request) }
|
187
269
|
return true
|
188
270
|
end
|
189
|
-
|
271
|
+
|
190
272
|
# Runs RequestLogAnalyzer
|
191
273
|
# 1. Call prepare on every aggregator
|
192
274
|
# 2. Generate requests from source object
|
@@ -196,10 +278,12 @@ module RequestLogAnalyzer
|
|
196
278
|
# 5. Call report on every aggregator
|
197
279
|
# 6. Finalize Source
|
198
280
|
def run!
|
199
|
-
|
281
|
+
|
282
|
+
# @aggregators.each{|agg| p agg}
|
283
|
+
|
200
284
|
@aggregators.each { |agg| agg.prepare }
|
201
285
|
install_signal_handlers
|
202
|
-
|
286
|
+
|
203
287
|
@source.each_request do |request|
|
204
288
|
break if @interrupted
|
205
289
|
aggregate_request(filter_request(request))
|
@@ -210,9 +294,9 @@ module RequestLogAnalyzer
|
|
210
294
|
@output.header
|
211
295
|
@aggregators.each { |agg| agg.report(@output) }
|
212
296
|
@output.footer
|
213
|
-
|
297
|
+
|
214
298
|
@source.finalize
|
215
|
-
|
299
|
+
|
216
300
|
if @output.io.kind_of?(File)
|
217
301
|
puts
|
218
302
|
puts "Report written to: " + File.expand_path(@output.io.path)
|
@@ -224,7 +308,7 @@ module RequestLogAnalyzer
|
|
224
308
|
@output.io.mail
|
225
309
|
end
|
226
310
|
end
|
227
|
-
|
311
|
+
|
228
312
|
def install_signal_handlers
|
229
313
|
Signal.trap("INT") do
|
230
314
|
handle_progress(:interrupted)
|
@@ -232,6 +316,6 @@ module RequestLogAnalyzer
|
|
232
316
|
@interrupted = true
|
233
317
|
end
|
234
318
|
end
|
235
|
-
|
319
|
+
|
236
320
|
end
|
237
321
|
end
|