ngmoco-request-log-analyzer 1.4.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +10 -0
- data/DESIGN.rdoc +41 -0
- data/LICENSE +20 -0
- data/README.rdoc +39 -0
- data/Rakefile +8 -0
- data/bin/request-log-analyzer +114 -0
- data/lib/cli/command_line_arguments.rb +301 -0
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +43 -0
- data/lib/cli/progressbar.rb +213 -0
- data/lib/cli/tools.rb +46 -0
- data/lib/request_log_analyzer.rb +44 -0
- data/lib/request_log_analyzer/aggregator.rb +49 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +29 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +175 -0
- data/lib/request_log_analyzer/controller.rb +332 -0
- data/lib/request_log_analyzer/database.rb +102 -0
- data/lib/request_log_analyzer/database/base.rb +115 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/file_format.rb +160 -0
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +71 -0
- data/lib/request_log_analyzer/file_format/apache.rb +141 -0
- data/lib/request_log_analyzer/file_format/merb.rb +67 -0
- data/lib/request_log_analyzer/file_format/rack.rb +11 -0
- data/lib/request_log_analyzer/file_format/rails.rb +176 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
- data/lib/request_log_analyzer/filter.rb +30 -0
- data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
- data/lib/request_log_analyzer/filter/field.rb +42 -0
- data/lib/request_log_analyzer/filter/timespan.rb +45 -0
- data/lib/request_log_analyzer/line_definition.rb +111 -0
- data/lib/request_log_analyzer/log_processor.rb +99 -0
- data/lib/request_log_analyzer/mailer.rb +62 -0
- data/lib/request_log_analyzer/output.rb +113 -0
- data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
- data/lib/request_log_analyzer/output/html.rb +184 -0
- data/lib/request_log_analyzer/request.rb +175 -0
- data/lib/request_log_analyzer/source.rb +72 -0
- data/lib/request_log_analyzer/source/database_loader.rb +87 -0
- data/lib/request_log_analyzer/source/log_parser.rb +274 -0
- data/lib/request_log_analyzer/tracker.rb +206 -0
- data/lib/request_log_analyzer/tracker/duration.rb +104 -0
- data/lib/request_log_analyzer/tracker/frequency.rb +95 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +107 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +81 -0
- data/lib/request_log_analyzer/tracker/traffic.rb +106 -0
- data/request-log-analyzer.gemspec +40 -0
- data/spec/database.yml +23 -0
- data/spec/fixtures/apache_combined.log +5 -0
- data/spec/fixtures/apache_common.log +10 -0
- data/spec/fixtures/decompression.log +12 -0
- data/spec/fixtures/decompression.log.bz2 +0 -0
- data/spec/fixtures/decompression.log.gz +0 -0
- data/spec/fixtures/decompression.log.zip +0 -0
- data/spec/fixtures/decompression.tar.gz +0 -0
- data/spec/fixtures/decompression.tgz +0 -0
- data/spec/fixtures/header_and_footer.log +6 -0
- data/spec/fixtures/merb.log +84 -0
- data/spec/fixtures/merb_prefixed.log +9 -0
- data/spec/fixtures/multiple_files_1.log +5 -0
- data/spec/fixtures/multiple_files_2.log +2 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/fixtures/rails_1x.log +59 -0
- data/spec/fixtures/rails_22.log +12 -0
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/spec/fixtures/syslog_1x.log +5 -0
- data/spec/fixtures/test_file_format.log +13 -0
- data/spec/fixtures/test_language_combined.log +14 -0
- data/spec/fixtures/test_order.log +16 -0
- data/spec/integration/command_line_usage_spec.rb +84 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +151 -0
- data/spec/lib/helpers.rb +52 -0
- data/spec/lib/macros.rb +18 -0
- data/spec/lib/matchers.rb +77 -0
- data/spec/lib/mocks.rb +76 -0
- data/spec/lib/testing_format.rb +46 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/aggregator/summarizer_spec.rb +26 -0
- data/spec/unit/controller/controller_spec.rb +41 -0
- data/spec/unit/controller/log_processor_spec.rb +18 -0
- data/spec/unit/database/base_class_spec.rb +183 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +133 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +49 -0
- data/spec/unit/file_format/apache_format_spec.rb +203 -0
- data/spec/unit/file_format/file_format_api_spec.rb +69 -0
- data/spec/unit/file_format/line_definition_spec.rb +75 -0
- data/spec/unit/file_format/merb_format_spec.rb +52 -0
- data/spec/unit/file_format/rails_format_spec.rb +164 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
- data/spec/unit/filter/field_filter_spec.rb +66 -0
- data/spec/unit/filter/filter_spec.rb +17 -0
- data/spec/unit/filter/timespan_filter_spec.rb +58 -0
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/request_spec.rb +111 -0
- data/spec/unit/source/log_parser_spec.rb +119 -0
- data/spec/unit/tracker/duration_tracker_spec.rb +130 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
- data/spec/unit/tracker/tracker_api_spec.rb +124 -0
- data/spec/unit/tracker/traffic_tracker_spec.rb +107 -0
- data/tasks/github-gem.rake +323 -0
- data/tasks/request_log_analyzer.rake +26 -0
- metadata +220 -0
@@ -0,0 +1,29 @@
|
|
1
|
+
module RequestLogAnalyzer::Aggregator
|
2
|
+
|
3
|
+
# Echo Aggregator. Writes everything to the screen when it is passed to this aggregator
|
4
|
+
class Echo < Base
|
5
|
+
|
6
|
+
attr_accessor :warnings
|
7
|
+
|
8
|
+
def prepare
|
9
|
+
@warnings = []
|
10
|
+
end
|
11
|
+
|
12
|
+
# Display every parsed line immediately to the terminal
|
13
|
+
def aggregate(request)
|
14
|
+
puts "\nRequest: " + request.lines.inspect
|
15
|
+
end
|
16
|
+
|
17
|
+
# Capture all warnings during parsing
|
18
|
+
def warning(type, message, lineno)
|
19
|
+
@warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}"
|
20
|
+
end
|
21
|
+
|
22
|
+
# Display every warning in the report when finished parsing
|
23
|
+
def report(output)
|
24
|
+
output.title("Warnings during parsing")
|
25
|
+
@warnings.each { |w| output.puts(w) }
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,175 @@
|
|
1
|
+
module RequestLogAnalyzer::Aggregator
|
2
|
+
|
3
|
+
class Summarizer < Base
|
4
|
+
|
5
|
+
class Definer
|
6
|
+
|
7
|
+
attr_reader :trackers
|
8
|
+
|
9
|
+
# Initialize tracker array
|
10
|
+
def initialize
|
11
|
+
@trackers = []
|
12
|
+
end
|
13
|
+
|
14
|
+
# Initialize tracker summarizer by duping the trackers of another summarizer
|
15
|
+
# <tt>other</tt> The other Summarizer
|
16
|
+
def initialize_copy(other)
|
17
|
+
@trackers = other.trackers.dup
|
18
|
+
end
|
19
|
+
|
20
|
+
# Drop all trackers
|
21
|
+
def reset!
|
22
|
+
@trackers = []
|
23
|
+
end
|
24
|
+
|
25
|
+
# Include missing trackers through method missing.
|
26
|
+
def method_missing(tracker_method, *args)
|
27
|
+
track(tracker_method, *args)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Track the frequency of a specific category
|
31
|
+
# <tt>category_field</tt> Field to track
|
32
|
+
# <tt>options</tt> options are passed to new frequency tracker
|
33
|
+
def frequency(category_field, options = {})
|
34
|
+
if category_field.kind_of?(Symbol)
|
35
|
+
track(:frequency, options.merge(:category => category_field))
|
36
|
+
elsif category_field.kind_of?(Hash)
|
37
|
+
track(:frequency, category_field.merge(options))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# Track the duration of a specific category
|
42
|
+
# <tt>duration_field</tt> Field to track
|
43
|
+
# <tt>options</tt> options are passed to new frequency tracker
|
44
|
+
def duration(duration_field, options = {})
|
45
|
+
if duration_field.kind_of?(Symbol)
|
46
|
+
track(:duration, options.merge(:duration => duration_field))
|
47
|
+
elsif duration_field.kind_of?(Hash)
|
48
|
+
track(:duration, duration_field.merge(options))
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# Helper function to initialize a tracker and add it to the tracker array.
|
53
|
+
# <tt>tracker_class</tt> The class to include
|
54
|
+
# <tt>optiont</tt> The options to pass to the trackers.
|
55
|
+
def track(tracker_klass, options = {})
|
56
|
+
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer::to_camelcase(tracker_klass)) if tracker_klass.kind_of?(Symbol)
|
57
|
+
@trackers << tracker_klass.new(options)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
attr_reader :trackers
|
62
|
+
attr_reader :warnings_encountered
|
63
|
+
|
64
|
+
# Initialize summarizer.
|
65
|
+
# Generate trackers from speciefied source.file_format.report_trackers and set them up
|
66
|
+
def initialize(source, options = {})
|
67
|
+
super(source, options)
|
68
|
+
@warnings_encountered = {}
|
69
|
+
@trackers = source.file_format.report_trackers
|
70
|
+
setup
|
71
|
+
end
|
72
|
+
|
73
|
+
def setup
|
74
|
+
end
|
75
|
+
|
76
|
+
# Call prepare on all trackers.
|
77
|
+
def prepare
|
78
|
+
raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
|
79
|
+
@trackers.each { |tracker| tracker.prepare }
|
80
|
+
end
|
81
|
+
|
82
|
+
# Pass all requests to trackers and let them update if necessary.
|
83
|
+
# <tt>request</tt> The request to pass.
|
84
|
+
def aggregate(request)
|
85
|
+
@trackers.each do |tracker|
|
86
|
+
tracker.update(request) if tracker.should_update?(request)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# Call finalize on all trackers. Saves a YAML dump if this is set in the options.
|
91
|
+
def finalize
|
92
|
+
@trackers.each { |tracker| tracker.finalize }
|
93
|
+
save_results_dump(options[:yaml]) if options[:yaml]
|
94
|
+
end
|
95
|
+
|
96
|
+
# Saves the results of all the trackers in YAML format to a file.
|
97
|
+
# <tt>filename</tt> The file to store the YAML dump in.
|
98
|
+
def save_results_dump(filename)
|
99
|
+
File.open(filename, 'w') { |file| file.write(to_yaml) }
|
100
|
+
end
|
101
|
+
|
102
|
+
# Exports all the tracker results to YAML. It will call the to_yaml_object method
|
103
|
+
# for every tracker and combines these into a single YAML export.
|
104
|
+
def to_yaml
|
105
|
+
require 'yaml'
|
106
|
+
trackers_export = @trackers.inject({}) do |export, tracker|
|
107
|
+
export[tracker.title] = tracker.to_yaml_object; export
|
108
|
+
end
|
109
|
+
YAML::dump(trackers_export)
|
110
|
+
end
|
111
|
+
|
112
|
+
# Call report on all trackers.
|
113
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
114
|
+
def report(output)
|
115
|
+
report_header(output)
|
116
|
+
if source.parsed_requests > 0
|
117
|
+
@trackers.each { |tracker| tracker.report(output) }
|
118
|
+
else
|
119
|
+
output.puts
|
120
|
+
output.puts('There were no requests analyzed.')
|
121
|
+
end
|
122
|
+
report_footer(output)
|
123
|
+
end
|
124
|
+
|
125
|
+
# Generate report header.
|
126
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
127
|
+
def report_header(output)
|
128
|
+
output.title("Request summary")
|
129
|
+
|
130
|
+
output.with_style(:cell_separator => false) do
|
131
|
+
output.table({:width => 20}, {:font => :bold}) do |rows|
|
132
|
+
rows << ['Parsed lines:', source.parsed_lines]
|
133
|
+
rows << ['Skipped lines:', source.skipped_lines]
|
134
|
+
rows << ['Parsed requests:', source.parsed_requests]
|
135
|
+
rows << ['Skipped requests:', source.skipped_requests]
|
136
|
+
rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
|
137
|
+
end
|
138
|
+
end
|
139
|
+
output << "\n"
|
140
|
+
end
|
141
|
+
|
142
|
+
# Generate report footer.
|
143
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
144
|
+
def report_footer(output)
|
145
|
+
if has_log_ordering_warnings?
|
146
|
+
output.title("Parse warnings")
|
147
|
+
|
148
|
+
output.puts "Parseable lines were ancountered without a header line before it. It"
|
149
|
+
output.puts "could be that logging is not setup correctly for your application."
|
150
|
+
output.puts "Visit this website for logging configuration tips:"
|
151
|
+
output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
|
152
|
+
output.puts
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
# Returns true if there were any warnings generated by the trackers
|
157
|
+
def has_warnings?
|
158
|
+
@warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
|
159
|
+
end
|
160
|
+
|
161
|
+
# Returns true if there were any log ordering warnings
|
162
|
+
def has_log_ordering_warnings?
|
163
|
+
@warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
|
164
|
+
end
|
165
|
+
|
166
|
+
# Store an encountered warning
|
167
|
+
# <tt>type</tt> Type of warning
|
168
|
+
# <tt>message</tt> Warning message
|
169
|
+
# <tt>lineno</tt> The line on which the error was encountered
|
170
|
+
def warning(type, message, lineno)
|
171
|
+
@warnings_encountered[type] ||= 0
|
172
|
+
@warnings_encountered[type] += 1
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
@@ -0,0 +1,332 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
|
3
|
+
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
4
|
+
# requested file format and connect it with sources and aggregators.
|
5
|
+
#
|
6
|
+
# Sources are streams or files from which the requests will be parsed.
|
7
|
+
# Aggregators will handle every passed request to yield a meaningfull results.
|
8
|
+
#
|
9
|
+
# - Use the build -function to build a new controller instance.
|
10
|
+
# - Use the run! method to start the parser and send the requests to the aggregators.
|
11
|
+
#
|
12
|
+
# Note that the order of sources can be imported if you have log files than succeed
|
13
|
+
# eachother. Requests that span over succeeding files will be parsed correctly if the
|
14
|
+
# sources are registered in the correct order. This can be helpful to parse requests
|
15
|
+
# from several logrotated log files.
|
16
|
+
class Controller
|
17
|
+
|
18
|
+
attr_reader :source, :filters, :aggregators, :output, :options
|
19
|
+
|
20
|
+
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
21
|
+
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
22
|
+
def self.build_from_arguments(arguments)
|
23
|
+
|
24
|
+
options = {}
|
25
|
+
|
26
|
+
# Copy fields
|
27
|
+
options[:database] = arguments[:database]
|
28
|
+
options[:reset_database] = arguments[:reset_database]
|
29
|
+
options[:debug] = arguments[:debug]
|
30
|
+
options[:yaml] = arguments[:dump]
|
31
|
+
options[:mail] = arguments[:mail]
|
32
|
+
options[:no_progress] = arguments[:no_progress]
|
33
|
+
options[:format] = arguments[:format]
|
34
|
+
options[:output] = arguments[:output]
|
35
|
+
options[:file] = arguments[:file]
|
36
|
+
options[:format] = arguments[:format]
|
37
|
+
options[:after] = arguments[:after]
|
38
|
+
options[:before] = arguments[:before]
|
39
|
+
options[:reject] = arguments[:reject]
|
40
|
+
options[:select] = arguments[:select]
|
41
|
+
options[:boring] = arguments[:boring]
|
42
|
+
options[:aggregator] = arguments[:aggregator]
|
43
|
+
options[:report_width] = arguments[:report_width]
|
44
|
+
options[:report_sort] = arguments[:report_sort]
|
45
|
+
options[:report_amount] = arguments[:report_amount]
|
46
|
+
|
47
|
+
# Apache format workaround
|
48
|
+
if arguments[:rails_format]
|
49
|
+
options[:format] = {:rails => arguments[:rails_format]}
|
50
|
+
elsif arguments[:apache_format]
|
51
|
+
options[:format] = {:apache => arguments[:apache_format]}
|
52
|
+
end
|
53
|
+
|
54
|
+
# Register sources
|
55
|
+
if arguments.parameters.length == 1
|
56
|
+
file = arguments.parameters[0]
|
57
|
+
if file == '-' || file == 'STDIN'
|
58
|
+
options.store(:source_files, $stdin)
|
59
|
+
elsif File.exist?(file)
|
60
|
+
options.store(:source_files, file)
|
61
|
+
else
|
62
|
+
puts "File not found: #{file}"
|
63
|
+
exit(0)
|
64
|
+
end
|
65
|
+
else
|
66
|
+
options.store(:source_files, arguments.parameters)
|
67
|
+
end
|
68
|
+
|
69
|
+
build(options)
|
70
|
+
end
|
71
|
+
|
72
|
+
# Build a new controller.
|
73
|
+
# Returns a new RequestLogAnalyzer::Controller object.
|
74
|
+
#
|
75
|
+
# Options
|
76
|
+
# * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
77
|
+
# * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY! - Defaults to [:summarizer]).
|
78
|
+
# * <tt>:boring</tt> Do not show color on STDOUT (Defaults to false).
|
79
|
+
# * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time or a String in "YYYY-MM-DD hh:mm:ss" format)
|
80
|
+
# * <tt>:database</tt> Database file to insert encountered requests to.
|
81
|
+
# * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
|
82
|
+
# * <tt>:file</tt> Filestring, File or StringIO.
|
83
|
+
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. (Defaults to :rails).
|
84
|
+
# * <tt>:mail</tt> Email the results to this email address.
|
85
|
+
# * <tt>:no_progress</tt> Do not display the progress bar (increases speed).
|
86
|
+
# * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
|
87
|
+
# * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
|
88
|
+
# * <tt>:report_width</tt> Width or reports in characters. (Defaults to 80)
|
89
|
+
# * <tt>:reset_database</tt> Reset the database before starting.
|
90
|
+
# * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
|
91
|
+
# * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
|
92
|
+
# * <tt>:yaml</tt> Output to YAML file.
|
93
|
+
#
|
94
|
+
# === Example
|
95
|
+
# RequestLogAnalyzer::Controller.build(
|
96
|
+
# :output => :HTML,
|
97
|
+
# :mail => 'root@localhost',
|
98
|
+
# :after => Time.now - 24*60*60,
|
99
|
+
# :source_files => '/var/log/passenger.log'
|
100
|
+
# ).run!
|
101
|
+
#
|
102
|
+
# === Todo
|
103
|
+
# * Check if defaults work (Aggregator defaults seem wrong).
|
104
|
+
# * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
|
105
|
+
def self.build(options)
|
106
|
+
# Defaults
|
107
|
+
options[:output] ||= 'fixed_width'
|
108
|
+
options[:format] ||= :rails
|
109
|
+
options[:aggregator] ||= [:summarizer]
|
110
|
+
options[:report_width] ||= 80
|
111
|
+
options[:report_amount] ||= 20
|
112
|
+
options[:report_sort] ||= 'sum,mean'
|
113
|
+
options[:boring] ||= false
|
114
|
+
|
115
|
+
# Deprecation warnings
|
116
|
+
if options[:dump] && options[:yaml].blank?
|
117
|
+
warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead."
|
118
|
+
options[:yaml] = options[:dump]
|
119
|
+
end
|
120
|
+
|
121
|
+
# Set the output class
|
122
|
+
output_args = {}
|
123
|
+
output_object = nil
|
124
|
+
if options[:output].is_a? Class
|
125
|
+
output_class = options[:output]
|
126
|
+
else
|
127
|
+
output_class = RequestLogAnalyzer::Output::const_get(options[:output])
|
128
|
+
end
|
129
|
+
|
130
|
+
output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
|
131
|
+
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
132
|
+
|
133
|
+
if options[:file]
|
134
|
+
output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
|
135
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
136
|
+
elsif options[:mail]
|
137
|
+
output_object = RequestLogAnalyzer::Mailer.new(options[:mail])
|
138
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
139
|
+
else
|
140
|
+
output_object = STDOUT
|
141
|
+
output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
|
142
|
+
:characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
|
143
|
+
end
|
144
|
+
|
145
|
+
output_instance = output_class.new(output_object, output_args)
|
146
|
+
|
147
|
+
# Create the controller with the correct file format
|
148
|
+
if options[:format].kind_of?(Hash)
|
149
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
150
|
+
else
|
151
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
152
|
+
end
|
153
|
+
|
154
|
+
# Kickstart the controller
|
155
|
+
controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
|
156
|
+
{ :output => output_instance,
|
157
|
+
:database => options[:database], # FUGLY!
|
158
|
+
:yaml => options[:yaml],
|
159
|
+
:reset_database => options[:reset_database],
|
160
|
+
:no_progress => options[:no_progress]})
|
161
|
+
|
162
|
+
# register filters
|
163
|
+
if options[:after] || options[:before]
|
164
|
+
filter_options = {}
|
165
|
+
[:after, :before].each do |filter|
|
166
|
+
case options[filter]
|
167
|
+
when Date, DateTime, Time
|
168
|
+
filter_options[filter] = options[filter]
|
169
|
+
when String
|
170
|
+
filter_options[filter] = DateTime.parse(options[filter])
|
171
|
+
end
|
172
|
+
end
|
173
|
+
controller.add_filter(:timespan, filter_options)
|
174
|
+
end
|
175
|
+
|
176
|
+
if options[:reject]
|
177
|
+
options[:reject].each do |(field, value)|
|
178
|
+
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
if options[:reject]
|
183
|
+
options[:select].each do |(field, value)|
|
184
|
+
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
# register aggregators
|
189
|
+
options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
|
190
|
+
controller.add_aggregator(:summarizer) if options[:aggregator].empty?
|
191
|
+
controller.add_aggregator(:echo) if options[:debug]
|
192
|
+
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
193
|
+
|
194
|
+
file_format.setup_environment(controller)
|
195
|
+
return controller
|
196
|
+
end
|
197
|
+
|
198
|
+
# Builds a new Controller for the given log file format.
|
199
|
+
# <tt>format</tt> Logfile format. Defaults to :rails
|
200
|
+
# Options are passd on to the LogParser.
|
201
|
+
# * <tt>:database</tt> Database the controller should use.
|
202
|
+
# * <tt>:yaml</tt> Yaml Dump the contrller should use.
|
203
|
+
# * <tt>:output</tt> All report outputs get << through this output.
|
204
|
+
# * <tt>:no_progress</tt> No progress bar
|
205
|
+
def initialize(source, options = {})
|
206
|
+
|
207
|
+
@source = source
|
208
|
+
@options = options
|
209
|
+
@aggregators = []
|
210
|
+
@filters = []
|
211
|
+
@output = options[:output]
|
212
|
+
@interrupted = false
|
213
|
+
|
214
|
+
# Register the request format for this session after checking its validity
|
215
|
+
raise "Invalid file format!" unless @source.file_format.valid?
|
216
|
+
|
217
|
+
# Install event handlers for wrnings, progress updates and source changes
|
218
|
+
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
219
|
+
@source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
|
220
|
+
@source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
|
221
|
+
end
|
222
|
+
|
223
|
+
# Progress function.
|
224
|
+
# Expects :started with file, :progress with current line and :finished or :interrupted when done.
|
225
|
+
# <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
|
226
|
+
# <tt>value</tt> File or current line.
|
227
|
+
def handle_progress(message, value = nil)
|
228
|
+
case message
|
229
|
+
when :started
|
230
|
+
@progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT)
|
231
|
+
when :finished
|
232
|
+
@progress_bar.finish
|
233
|
+
@progress_bar = nil
|
234
|
+
when :interrupted
|
235
|
+
if @progress_bar
|
236
|
+
@progress_bar.halt
|
237
|
+
@progress_bar = nil
|
238
|
+
end
|
239
|
+
when :progress
|
240
|
+
@progress_bar.set(value)
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
# Source change handler
|
245
|
+
def handle_source_change(change, filename)
|
246
|
+
@aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
|
247
|
+
end
|
248
|
+
|
249
|
+
# Adds an aggregator to the controller. The aggregator will be called for every request
|
250
|
+
# that is parsed from the provided sources (see add_source)
|
251
|
+
def add_aggregator(agg)
|
252
|
+
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
|
253
|
+
@aggregators << agg.new(@source, @options)
|
254
|
+
end
|
255
|
+
|
256
|
+
alias :>> :add_aggregator
|
257
|
+
|
258
|
+
# Adds a request filter to the controller.
|
259
|
+
def add_filter(filter, filter_options = {})
|
260
|
+
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
|
261
|
+
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
262
|
+
end
|
263
|
+
|
264
|
+
# Push a request through the entire filterchain (@filters).
|
265
|
+
# <tt>request</tt> The request to filter.
|
266
|
+
# Returns the filtered request or nil.
|
267
|
+
def filter_request(request)
|
268
|
+
@filters.each do |filter|
|
269
|
+
request = filter.filter(request)
|
270
|
+
return nil if request.nil?
|
271
|
+
end
|
272
|
+
return request
|
273
|
+
end
|
274
|
+
|
275
|
+
# Push a request to all the aggregators (@aggregators).
|
276
|
+
# <tt>request</tt> The request to push to the aggregators.
|
277
|
+
def aggregate_request(request)
|
278
|
+
return false unless request
|
279
|
+
@aggregators.each { |agg| agg.aggregate(request) }
|
280
|
+
return true
|
281
|
+
end
|
282
|
+
|
283
|
+
# Runs RequestLogAnalyzer
|
284
|
+
# 1. Call prepare on every aggregator
|
285
|
+
# 2. Generate requests from source object
|
286
|
+
# 3. Filter out unwanted requests
|
287
|
+
# 4. Call aggregate for remaning requests on every aggregator
|
288
|
+
# 4. Call finalize on every aggregator
|
289
|
+
# 5. Call report on every aggregator
|
290
|
+
# 6. Finalize Source
|
291
|
+
def run!
|
292
|
+
|
293
|
+
# @aggregators.each{|agg| p agg}
|
294
|
+
|
295
|
+
@aggregators.each { |agg| agg.prepare }
|
296
|
+
install_signal_handlers
|
297
|
+
|
298
|
+
@source.each_request do |request|
|
299
|
+
break if @interrupted
|
300
|
+
aggregate_request(filter_request(request))
|
301
|
+
end
|
302
|
+
|
303
|
+
@aggregators.each { |agg| agg.finalize }
|
304
|
+
|
305
|
+
@output.header
|
306
|
+
@aggregators.each { |agg| agg.report(@output) }
|
307
|
+
@output.footer
|
308
|
+
|
309
|
+
@source.finalize
|
310
|
+
|
311
|
+
if @output.io.kind_of?(File)
|
312
|
+
puts
|
313
|
+
puts "Report written to: " + File.expand_path(@output.io.path)
|
314
|
+
puts "Need an expert to analyze your application?"
|
315
|
+
puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
|
316
|
+
puts "Thanks for using request-log-analyzer!"
|
317
|
+
@output.io.close
|
318
|
+
elsif @output.io.kind_of?(RequestLogAnalyzer::Mailer)
|
319
|
+
@output.io.mail
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
def install_signal_handlers
|
324
|
+
Signal.trap("INT") do
|
325
|
+
handle_progress(:interrupted)
|
326
|
+
puts "Caught interrupt! Stopping parsing..."
|
327
|
+
@interrupted = true
|
328
|
+
end
|
329
|
+
end
|
330
|
+
|
331
|
+
end
|
332
|
+
end
|