request-log-analyzer 1.0.2 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +10 -0
- data/DESIGN.rdoc +41 -0
- data/LICENSE +4 -4
- data/README.rdoc +38 -0
- data/Rakefile +6 -3
- data/bin/request-log-analyzer +70 -72
- data/lib/cli/command_line_arguments.rb +53 -53
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +43 -0
- data/lib/cli/progressbar.rb +166 -189
- data/lib/cli/tools.rb +49 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
- data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
- data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
- data/lib/request_log_analyzer/controller.rb +251 -98
- data/lib/request_log_analyzer/database/base.rb +114 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +102 -0
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
- data/lib/request_log_analyzer/file_format/apache.rb +147 -0
- data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
- data/lib/request_log_analyzer/file_format/merb.rb +65 -29
- data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
- data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
- data/lib/request_log_analyzer/file_format/rack.rb +9 -0
- data/lib/request_log_analyzer/file_format/rails.rb +164 -78
- data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
- data/lib/request_log_analyzer/file_format.rb +252 -58
- data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
- data/lib/request_log_analyzer/filter/field.rb +19 -13
- data/lib/request_log_analyzer/filter/timespan.rb +25 -12
- data/lib/request_log_analyzer/filter.rb +30 -0
- data/lib/request_log_analyzer/line_definition.rb +69 -96
- data/lib/request_log_analyzer/log_processor.rb +31 -53
- data/lib/request_log_analyzer/mailer.rb +65 -0
- data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
- data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
- data/lib/request_log_analyzer/output/html.rb +187 -0
- data/lib/request_log_analyzer/output.rb +117 -0
- data/lib/request_log_analyzer/request.rb +125 -40
- data/lib/request_log_analyzer/source/database_loader.rb +87 -0
- data/lib/request_log_analyzer/source/log_parser.rb +297 -0
- data/lib/request_log_analyzer/source.rb +72 -0
- data/lib/request_log_analyzer/tracker/duration.rb +28 -64
- data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
- data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
- data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
- data/lib/request_log_analyzer/tracker.rb +101 -0
- data/lib/request_log_analyzer.rb +43 -13
- data/request-log-analyzer.gemspec +41 -0
- data/spec/database.yml +23 -0
- data/spec/fixtures/apache_combined.log +5 -0
- data/spec/fixtures/apache_common.log +10 -0
- data/spec/fixtures/decompression.log +12 -0
- data/spec/fixtures/decompression.log.bz2 +0 -0
- data/spec/fixtures/decompression.log.gz +0 -0
- data/spec/fixtures/decompression.log.zip +0 -0
- data/spec/fixtures/decompression.tar.gz +0 -0
- data/spec/fixtures/decompression.tgz +0 -0
- data/spec/fixtures/header_and_footer.log +6 -0
- data/spec/fixtures/merb_prefixed.log +9 -0
- data/spec/fixtures/mysql_slow_query.log +110 -0
- data/spec/fixtures/postgresql.log +2980 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/fixtures/sinatra.log +99 -0
- data/spec/integration/command_line_usage_spec.rb +84 -0
- data/spec/integration/mailer_spec.rb +179 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +152 -0
- data/spec/lib/helpers.rb +72 -0
- data/spec/lib/macros.rb +18 -0
- data/spec/lib/matchers.rb +77 -0
- data/spec/lib/mocks.rb +77 -0
- data/spec/lib/testing_format.rb +46 -0
- data/spec/spec_helper.rb +16 -59
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/aggregator/summarizer_spec.rb +26 -0
- data/spec/unit/controller/controller_spec.rb +41 -0
- data/spec/unit/controller/log_processor_spec.rb +18 -0
- data/spec/unit/database/base_class_spec.rb +183 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +133 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
- data/spec/unit/file_format/apache_format_spec.rb +203 -0
- data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
- data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
- data/spec/unit/file_format/file_format_api_spec.rb +69 -0
- data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
- data/spec/unit/file_format/line_definition_spec.rb +75 -0
- data/spec/unit/file_format/merb_format_spec.rb +52 -0
- data/spec/unit/file_format/mysql_format_spec.rb +154 -0
- data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
- data/spec/unit/file_format/rack_format_spec.rb +50 -0
- data/spec/unit/file_format/rails3_format_spec.rb +120 -0
- data/spec/unit/file_format/rails_format_spec.rb +180 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
- data/spec/unit/filter/field_filter_spec.rb +66 -0
- data/spec/unit/filter/filter_spec.rb +17 -0
- data/spec/unit/filter/timespan_filter_spec.rb +58 -0
- data/spec/unit/mailer_spec.rb +42 -0
- data/spec/unit/request_spec.rb +111 -0
- data/spec/unit/source/log_parser_spec.rb +119 -0
- data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
- data/spec/unit/tracker/tracker_api_spec.rb +125 -0
- data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
- data/tasks/github-gem.rake +290 -139
- data/tasks/request_log_analyzer.rake +23 -7
- metadata +221 -94
- data/DESIGN +0 -14
- data/HACKING +0 -7
- data/README.textile +0 -36
- data/lib/cli/bashcolorizer.rb +0 -60
- data/lib/request_log_analyzer/aggregator/database.rb +0 -148
- data/lib/request_log_analyzer/filter/base.rb +0 -29
- data/lib/request_log_analyzer/log_parser.rb +0 -173
- data/lib/request_log_analyzer/source/base.rb +0 -42
- data/lib/request_log_analyzer/source/log_file.rb +0 -170
- data/lib/request_log_analyzer/tracker/base.rb +0 -54
- data/lib/request_log_analyzer/tracker/category.rb +0 -71
- data/spec/controller_spec.rb +0 -40
- data/spec/database_inserter_spec.rb +0 -101
- data/spec/file_format_spec.rb +0 -78
- data/spec/file_formats/spec_format.rb +0 -26
- data/spec/filter_spec.rb +0 -137
- data/spec/line_definition_spec.rb +0 -124
- data/spec/log_parser_spec.rb +0 -68
- data/spec/log_processor_spec.rb +0 -57
- data/spec/merb_format_spec.rb +0 -38
- data/spec/rails_format_spec.rb +0 -76
- data/spec/request_spec.rb +0 -72
- data/spec/summarizer_spec.rb +0 -9
- data/tasks/rspec.rake +0 -6
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
module RequestLogAnalyzer
|
|
2
|
-
|
|
2
|
+
|
|
3
3
|
# The RequestLogAnalyzer::Controller class creates a LogParser instance for the
|
|
4
|
-
# requested file format
|
|
4
|
+
# requested file format and connect it with sources and aggregators.
|
|
5
5
|
#
|
|
6
6
|
# Sources are streams or files from which the requests will be parsed.
|
|
7
7
|
# Aggregators will handle every passed request to yield a meaningfull results.
|
|
8
8
|
#
|
|
9
|
-
# - Use the build-function to build a controller instance
|
|
10
|
-
# - Use add_aggregator to register a new aggregator
|
|
11
|
-
# - Use add_source to register a new aggregator
|
|
9
|
+
# - Use the build -function to build a new controller instance.
|
|
12
10
|
# - Use the run! method to start the parser and send the requests to the aggregators.
|
|
13
11
|
#
|
|
14
12
|
# Note that the order of sources can be imported if you have log files than succeed
|
|
@@ -17,35 +15,50 @@ module RequestLogAnalyzer
|
|
|
17
15
|
# from several logrotated log files.
|
|
18
16
|
class Controller
|
|
19
17
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
attr_reader :aggregators
|
|
23
|
-
attr_reader :filters
|
|
24
|
-
attr_reader :log_parser
|
|
25
|
-
attr_reader :source
|
|
26
|
-
attr_reader :output
|
|
27
|
-
attr_reader :options
|
|
18
|
+
attr_reader :source, :filters, :aggregators, :output, :options
|
|
28
19
|
|
|
29
20
|
# Builds a RequestLogAnalyzer::Controller given parsed command line arguments
|
|
30
21
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
options[:database]
|
|
37
|
-
options[:
|
|
38
|
-
options[:
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
22
|
+
def self.build_from_arguments(arguments)
|
|
23
|
+
|
|
24
|
+
options = {}
|
|
25
|
+
|
|
26
|
+
# Copy fields
|
|
27
|
+
options[:database] = arguments[:database]
|
|
28
|
+
options[:reset_database] = arguments[:reset_database]
|
|
29
|
+
options[:debug] = arguments[:debug]
|
|
30
|
+
options[:yaml] = arguments[:yaml] || arguments[:dump]
|
|
31
|
+
options[:mail] = arguments[:mail]
|
|
32
|
+
options[:no_progress] = arguments[:no_progress]
|
|
33
|
+
options[:format] = arguments[:format]
|
|
34
|
+
options[:output] = arguments[:output].downcase
|
|
35
|
+
options[:file] = arguments[:file]
|
|
36
|
+
options[:after] = arguments[:after]
|
|
37
|
+
options[:before] = arguments[:before]
|
|
38
|
+
options[:reject] = arguments[:reject]
|
|
39
|
+
options[:select] = arguments[:select]
|
|
40
|
+
options[:boring] = arguments[:boring]
|
|
41
|
+
options[:aggregator] = arguments[:aggregator]
|
|
42
|
+
options[:report_width] = arguments[:report_width]
|
|
43
|
+
options[:report_sort] = arguments[:report_sort]
|
|
44
|
+
options[:report_amount] = arguments[:report_amount]
|
|
45
|
+
options[:mailhost] = arguments[:mailhost]
|
|
46
|
+
|
|
47
|
+
# Apache format workaround
|
|
48
|
+
if arguments[:rails_format]
|
|
49
|
+
options[:format] = {:rails => arguments[:rails_format]}
|
|
50
|
+
elsif arguments[:apache_format]
|
|
51
|
+
options[:format] = {:apache => arguments[:apache_format]}
|
|
43
52
|
end
|
|
44
|
-
|
|
45
|
-
#
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
53
|
+
|
|
54
|
+
# Handle output format casing
|
|
55
|
+
if options[:output].class == String
|
|
56
|
+
options[:output] = 'FancyHTML' if options[:output] =~ /^fancy_?html$/i
|
|
57
|
+
options[:output] = 'HTML' if options[:output] =~ /^html$/i
|
|
58
|
+
options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Register sources
|
|
49
62
|
if arguments.parameters.length == 1
|
|
50
63
|
file = arguments.parameters[0]
|
|
51
64
|
if file == '-' || file == 'STDIN'
|
|
@@ -60,50 +73,157 @@ module RequestLogAnalyzer
|
|
|
60
73
|
options.store(:source_files, arguments.parameters)
|
|
61
74
|
end
|
|
62
75
|
|
|
63
|
-
|
|
76
|
+
# Guess file format
|
|
77
|
+
if !options[:format] && options[:source_files]
|
|
78
|
+
options[:format] = :rails # Default
|
|
79
|
+
|
|
80
|
+
if options[:source_files] != $stdin
|
|
81
|
+
if options[:source_files].class == String
|
|
82
|
+
options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files])
|
|
64
83
|
|
|
65
|
-
|
|
84
|
+
elsif options[:source_files].class == Array && options[:source_files].first != $stdin
|
|
85
|
+
options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files].first)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
66
89
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
90
|
+
build(options)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Build a new controller.
|
|
94
|
+
# Returns a new RequestLogAnalyzer::Controller object.
|
|
95
|
+
#
|
|
96
|
+
# Options
|
|
97
|
+
# * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
|
|
98
|
+
# * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY! - Defaults to [:summarizer]).
|
|
99
|
+
# * <tt>:boring</tt> Do not show color on STDOUT (Defaults to false).
|
|
100
|
+
# * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time or a String in "YYYY-MM-DD hh:mm:ss" format)
|
|
101
|
+
# * <tt>:database</tt> Database file to insert encountered requests to.
|
|
102
|
+
# * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
|
|
103
|
+
# * <tt>:file</tt> Filestring, File or StringIO.
|
|
104
|
+
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
|
|
105
|
+
# * <tt>:mail</tt> Email the results to this email address.
|
|
106
|
+
# * <tt>:mailhost</tt> Email the results to this mail server.
|
|
107
|
+
# * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
|
|
108
|
+
# * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
|
|
109
|
+
# * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
|
|
110
|
+
# * <tt>:report_width</tt> Width of reports in characters for FixedWidth reports. (Defaults to 80)
|
|
111
|
+
# * <tt>:reset_database</tt> Reset the database before starting.
|
|
112
|
+
# * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
|
|
113
|
+
# * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
|
|
114
|
+
# * <tt>:yaml</tt> Output to YAML file.
|
|
115
|
+
#
|
|
116
|
+
# === Example
|
|
117
|
+
# RequestLogAnalyzer::Controller.build(
|
|
118
|
+
# :output => :HTML,
|
|
119
|
+
# :mail => 'root@localhost',
|
|
120
|
+
# :after => Time.now - 24*60*60,
|
|
121
|
+
# :source_files => '/var/log/passenger.log'
|
|
122
|
+
# ).run!
|
|
123
|
+
#
|
|
124
|
+
# === Todo
|
|
125
|
+
# * Check if defaults work (Aggregator defaults seem wrong).
|
|
126
|
+
# * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
|
|
127
|
+
def self.build(options)
|
|
128
|
+
# Defaults
|
|
129
|
+
options[:output] ||= 'FixedWidth'
|
|
130
|
+
options[:format] ||= :rails
|
|
131
|
+
options[:aggregator] ||= [:summarizer]
|
|
132
|
+
options[:report_width] ||= 80
|
|
133
|
+
options[:report_amount] ||= 20
|
|
134
|
+
options[:report_sort] ||= 'sum,mean'
|
|
135
|
+
options[:boring] ||= false
|
|
136
|
+
|
|
137
|
+
# Deprecation warnings
|
|
138
|
+
if options[:dump]
|
|
139
|
+
warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead."
|
|
140
|
+
options[:yaml] = options[:dump]
|
|
73
141
|
end
|
|
74
142
|
|
|
75
|
-
|
|
76
|
-
|
|
143
|
+
# Set the output class
|
|
144
|
+
output_args = {}
|
|
145
|
+
output_object = nil
|
|
146
|
+
if options[:output].is_a?(Class)
|
|
147
|
+
output_class = options[:output]
|
|
148
|
+
else
|
|
149
|
+
output_class = RequestLogAnalyzer::Output::const_get(options[:output])
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
|
|
153
|
+
output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
|
|
154
|
+
|
|
155
|
+
if options[:file]
|
|
156
|
+
output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
|
|
157
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
|
158
|
+
elsif options[:mail]
|
|
159
|
+
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
|
|
160
|
+
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
|
161
|
+
else
|
|
162
|
+
output_object = STDOUT
|
|
163
|
+
output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
|
|
164
|
+
:characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
|
|
77
165
|
end
|
|
78
166
|
|
|
79
|
-
|
|
80
|
-
|
|
167
|
+
output_instance = output_class.new(output_object, output_args)
|
|
168
|
+
|
|
169
|
+
# Create the controller with the correct file format
|
|
170
|
+
if options[:format].kind_of?(Hash)
|
|
171
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
|
|
172
|
+
else
|
|
173
|
+
file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Kickstart the controller
|
|
177
|
+
controller = Controller.new( RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
|
|
178
|
+
{ :output => output_instance,
|
|
179
|
+
:database => options[:database], # FUGLY!
|
|
180
|
+
:yaml => options[:yaml],
|
|
181
|
+
:reset_database => options[:reset_database],
|
|
182
|
+
:no_progress => options[:no_progress]})
|
|
183
|
+
|
|
184
|
+
# register filters
|
|
185
|
+
if options[:after] || options[:before]
|
|
186
|
+
filter_options = {}
|
|
187
|
+
[:after, :before].each do |filter|
|
|
188
|
+
case options[filter]
|
|
189
|
+
when Date, DateTime, Time
|
|
190
|
+
filter_options[filter] = options[filter]
|
|
191
|
+
when String
|
|
192
|
+
filter_options[filter] = DateTime.parse(options[filter])
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
controller.add_filter(:timespan, filter_options)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
if options[:reject]
|
|
199
|
+
options[:reject].each do |(field, value)|
|
|
200
|
+
controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
if options[:reject]
|
|
205
|
+
options[:select].each do |(field, value)|
|
|
206
|
+
controller.add_filter(:field, :mode => :select, :field => field, :value => value)
|
|
207
|
+
end
|
|
81
208
|
end
|
|
82
209
|
|
|
83
210
|
# register aggregators
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
controller.add_aggregator(:
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
# register the echo aggregator in debug mode
|
|
91
|
-
controller.add_aggregator(:echo) if arguments[:debug]
|
|
92
|
-
|
|
211
|
+
options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
|
|
212
|
+
controller.add_aggregator(:summarizer) if options[:aggregator].empty?
|
|
213
|
+
controller.add_aggregator(:echo) if options[:debug]
|
|
214
|
+
controller.add_aggregator(:database_inserter) if options[:database] && !options[:aggregator].include?('database')
|
|
215
|
+
|
|
93
216
|
file_format.setup_environment(controller)
|
|
94
|
-
|
|
95
217
|
return controller
|
|
96
|
-
end
|
|
218
|
+
end
|
|
97
219
|
|
|
98
220
|
# Builds a new Controller for the given log file format.
|
|
99
221
|
# <tt>format</tt> Logfile format. Defaults to :rails
|
|
100
222
|
# Options are passd on to the LogParser.
|
|
101
|
-
# * <tt>:aggregator</tt> Aggregator array.
|
|
102
223
|
# * <tt>:database</tt> Database the controller should use.
|
|
103
|
-
# * <tt>:
|
|
104
|
-
# * <tt>:silent</tt> Do not output any warnings.
|
|
105
|
-
# * <tt>:colorize</tt> Colorize output
|
|
224
|
+
# * <tt>:yaml</tt> Yaml Dump the contrller should use.
|
|
106
225
|
# * <tt>:output</tt> All report outputs get << through this output.
|
|
226
|
+
# * <tt>:no_progress</tt> No progress bar
|
|
107
227
|
def initialize(source, options = {})
|
|
108
228
|
|
|
109
229
|
@source = source
|
|
@@ -111,17 +231,17 @@ module RequestLogAnalyzer
|
|
|
111
231
|
@aggregators = []
|
|
112
232
|
@filters = []
|
|
113
233
|
@output = options[:output]
|
|
234
|
+
@interrupted = false
|
|
114
235
|
|
|
115
|
-
#
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
# Pass all warnings to every aggregator so they can do something useful with them.
|
|
119
|
-
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
|
|
236
|
+
# Register the request format for this session after checking its validity
|
|
237
|
+
raise "Invalid file format!" unless @source.file_format.valid?
|
|
120
238
|
|
|
121
|
-
#
|
|
122
|
-
@source.
|
|
239
|
+
# Install event handlers for wrnings, progress updates and source changes
|
|
240
|
+
@source.warning = lambda { |type, message, lineno| @aggregators.each { |agg| agg.warning(type, message, lineno) } }
|
|
241
|
+
@source.progress = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
|
|
242
|
+
@source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
|
|
123
243
|
end
|
|
124
|
-
|
|
244
|
+
|
|
125
245
|
# Progress function.
|
|
126
246
|
# Expects :started with file, :progress with current line and :finished or :interrupted when done.
|
|
127
247
|
# <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
|
|
@@ -129,7 +249,7 @@ module RequestLogAnalyzer
|
|
|
129
249
|
def handle_progress(message, value = nil)
|
|
130
250
|
case message
|
|
131
251
|
when :started
|
|
132
|
-
@progress_bar = ProgressBar.new(
|
|
252
|
+
@progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT)
|
|
133
253
|
when :finished
|
|
134
254
|
@progress_bar.finish
|
|
135
255
|
@progress_bar = nil
|
|
@@ -142,30 +262,46 @@ module RequestLogAnalyzer
|
|
|
142
262
|
@progress_bar.set(value)
|
|
143
263
|
end
|
|
144
264
|
end
|
|
145
|
-
|
|
146
|
-
#
|
|
265
|
+
|
|
266
|
+
# Source change handler
|
|
267
|
+
def handle_source_change(change, filename)
|
|
268
|
+
@aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# Adds an aggregator to the controller. The aggregator will be called for every request
|
|
147
272
|
# that is parsed from the provided sources (see add_source)
|
|
148
273
|
def add_aggregator(agg)
|
|
149
|
-
if agg.kind_of?(Symbol)
|
|
150
|
-
require File.dirname(__FILE__) + "/aggregator/#{agg}"
|
|
151
|
-
agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
|
|
152
|
-
end
|
|
153
|
-
|
|
274
|
+
agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
|
|
154
275
|
@aggregators << agg.new(@source, @options)
|
|
155
276
|
end
|
|
156
|
-
|
|
277
|
+
|
|
157
278
|
alias :>> :add_aggregator
|
|
158
|
-
|
|
279
|
+
|
|
159
280
|
# Adds a request filter to the controller.
|
|
160
281
|
def add_filter(filter, filter_options = {})
|
|
161
|
-
if filter.kind_of?(Symbol)
|
|
162
|
-
|
|
163
|
-
|
|
282
|
+
filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
|
|
283
|
+
@filters << filter.new(source.file_format, @options.merge(filter_options))
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Push a request through the entire filterchain (@filters).
|
|
287
|
+
# <tt>request</tt> The request to filter.
|
|
288
|
+
# Returns the filtered request or nil.
|
|
289
|
+
def filter_request(request)
|
|
290
|
+
@filters.each do |filter|
|
|
291
|
+
request = filter.filter(request)
|
|
292
|
+
return nil if request.nil?
|
|
164
293
|
end
|
|
165
|
-
|
|
166
|
-
|
|
294
|
+
return request
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Push a request to all the aggregators (@aggregators).
|
|
298
|
+
# <tt>request</tt> The request to push to the aggregators.
|
|
299
|
+
def aggregate_request(request)
|
|
300
|
+
return false unless request
|
|
301
|
+
@aggregators.each { |agg| agg.aggregate(request) }
|
|
302
|
+
return true
|
|
167
303
|
end
|
|
168
|
-
|
|
304
|
+
|
|
169
305
|
# Runs RequestLogAnalyzer
|
|
170
306
|
# 1. Call prepare on every aggregator
|
|
171
307
|
# 2. Generate requests from source object
|
|
@@ -175,27 +311,44 @@ module RequestLogAnalyzer
|
|
|
175
311
|
# 5. Call report on every aggregator
|
|
176
312
|
# 6. Finalize Source
|
|
177
313
|
def run!
|
|
178
|
-
|
|
179
|
-
@
|
|
314
|
+
|
|
315
|
+
# @aggregators.each{|agg| p agg}
|
|
316
|
+
|
|
180
317
|
@aggregators.each { |agg| agg.prepare }
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
end
|
|
187
|
-
rescue Interrupt => e
|
|
188
|
-
handle_progress(:interrupted)
|
|
189
|
-
puts "Caught interrupt! Stopped parsing."
|
|
318
|
+
install_signal_handlers
|
|
319
|
+
|
|
320
|
+
@source.each_request do |request|
|
|
321
|
+
break if @interrupted
|
|
322
|
+
aggregate_request(filter_request(request))
|
|
190
323
|
end
|
|
191
324
|
|
|
192
|
-
puts "\n"
|
|
193
|
-
|
|
194
325
|
@aggregators.each { |agg| agg.finalize }
|
|
195
|
-
|
|
196
|
-
|
|
326
|
+
|
|
327
|
+
@output.header
|
|
328
|
+
@aggregators.each { |agg| agg.report(@output) }
|
|
329
|
+
@output.footer
|
|
330
|
+
|
|
197
331
|
@source.finalize
|
|
332
|
+
|
|
333
|
+
if @output.io.kind_of?(File)
|
|
334
|
+
puts
|
|
335
|
+
puts "Report written to: " + File.expand_path(@output.io.path)
|
|
336
|
+
puts "Need an expert to analyze your application?"
|
|
337
|
+
puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
|
|
338
|
+
puts "Thanks for using request-log-analyzer!"
|
|
339
|
+
@output.io.close
|
|
340
|
+
elsif @output.io.kind_of?(RequestLogAnalyzer::Mailer)
|
|
341
|
+
@output.io.mail
|
|
342
|
+
end
|
|
198
343
|
end
|
|
199
|
-
|
|
344
|
+
|
|
345
|
+
def install_signal_handlers
|
|
346
|
+
Signal.trap("INT") do
|
|
347
|
+
handle_progress(:interrupted)
|
|
348
|
+
puts "Caught interrupt! Stopping parsing..."
|
|
349
|
+
@interrupted = true
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
|
|
200
353
|
end
|
|
201
|
-
end
|
|
354
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
2
|
+
|
|
3
|
+
self.abstract_class = true
|
|
4
|
+
|
|
5
|
+
def <=>(other)
|
|
6
|
+
if (source_id.nil? && other.source_id.nil?) || (source_comparison = source_id <=> other.source_id) == 0
|
|
7
|
+
lineno <=> other.lineno
|
|
8
|
+
else
|
|
9
|
+
source_comparison
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def line_type
|
|
14
|
+
self.class.name.underscore.gsub(/_line$/, '').to_sym
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
class_inheritable_accessor :line_definition
|
|
18
|
+
cattr_accessor :database
|
|
19
|
+
|
|
20
|
+
def self.subclass_from_line_definition(definition)
|
|
21
|
+
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
|
22
|
+
klass.set_table_name("#{definition.name}_lines")
|
|
23
|
+
|
|
24
|
+
klass.line_definition = definition
|
|
25
|
+
|
|
26
|
+
# Set relations with requests and sources table
|
|
27
|
+
klass.belongs_to :request
|
|
28
|
+
klass.belongs_to :source
|
|
29
|
+
|
|
30
|
+
# Serialize complex fields into the database
|
|
31
|
+
definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
|
|
32
|
+
klass.send(:serialize, capture[:name], Hash)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
RequestLogAnalyzer::Database::Request.has_many "#{definition.name}_lines".to_sym
|
|
36
|
+
RequestLogAnalyzer::Database::Source.has_many "#{definition.name}_lines".to_sym
|
|
37
|
+
|
|
38
|
+
return klass
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.subclass_from_table(table)
|
|
42
|
+
raise "Table #{table} not found!" unless database.connection.table_exists?(table)
|
|
43
|
+
|
|
44
|
+
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
|
45
|
+
klass.set_table_name(table)
|
|
46
|
+
|
|
47
|
+
if klass.column_names.include?('request_id')
|
|
48
|
+
klass.belongs_to :request
|
|
49
|
+
RequestLogAnalyzer::Database::Request.has_many table.to_sym
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
if klass.column_names.include?('source_id')
|
|
53
|
+
klass.belongs_to :source
|
|
54
|
+
RequestLogAnalyzer::Database::Source.has_many table.to_sym
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
return klass
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def self.drop_table!
|
|
61
|
+
database.connection.remove_index(self.table_name, [:source_id]) rescue nil
|
|
62
|
+
database.connection.remove_index(self.table_name, [:request_id]) rescue nil
|
|
63
|
+
database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.create_table!
|
|
67
|
+
raise "No line_definition available to base table schema on!" unless self.line_definition
|
|
68
|
+
|
|
69
|
+
unless table_exists?
|
|
70
|
+
database.connection.create_table(table_name.to_sym) do |t|
|
|
71
|
+
|
|
72
|
+
# Default fields
|
|
73
|
+
t.column :request_id, :integer
|
|
74
|
+
t.column :source_id, :integer
|
|
75
|
+
t.column :lineno, :integer
|
|
76
|
+
|
|
77
|
+
line_definition.captures.each do |capture|
|
|
78
|
+
# Add a field for every capture
|
|
79
|
+
t.column(capture[:name], column_type(capture[:type]))
|
|
80
|
+
|
|
81
|
+
# If the capture provides other field as well, create columns for them, too
|
|
82
|
+
capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Add indices to table for more speedy querying
|
|
87
|
+
database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
|
|
88
|
+
database.connection.add_index(self.table_name.to_sym, [:source_id]) # rescue
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Function to determine the column type for a field
|
|
93
|
+
# TODO: make more robust / include in file-format definition
|
|
94
|
+
def self.column_type(type_indicator)
|
|
95
|
+
case type_indicator
|
|
96
|
+
when :eval; :text
|
|
97
|
+
when :hash; :text
|
|
98
|
+
when :text; :text
|
|
99
|
+
when :string; :string
|
|
100
|
+
when :sec; :double
|
|
101
|
+
when :msec; :double
|
|
102
|
+
when :duration; :double
|
|
103
|
+
when :float; :double
|
|
104
|
+
when :double; :double
|
|
105
|
+
when :integer; :integer
|
|
106
|
+
when :int; :int
|
|
107
|
+
when :timestamp; :datetime
|
|
108
|
+
when :datetime; :datetime
|
|
109
|
+
when :date; :date
|
|
110
|
+
else :string
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
module RequestLogAnalyzer::Database::Connection
|
|
2
|
+
|
|
3
|
+
def self.from_string(string)
|
|
4
|
+
hash = {}
|
|
5
|
+
if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/
|
|
6
|
+
string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value }
|
|
7
|
+
elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/
|
|
8
|
+
hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = $1, $2, $3, $4, $5
|
|
9
|
+
hash.delete_if { |k, v| v.nil? }
|
|
10
|
+
end
|
|
11
|
+
return hash.empty? ? nil : hash
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def connect(connection_identifier)
|
|
15
|
+
if connection_identifier.kind_of?(Hash)
|
|
16
|
+
RequestLogAnalyzer::Database::Base.establish_connection(connection_identifier)
|
|
17
|
+
elsif connection_identifier == ':memory:'
|
|
18
|
+
RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => ':memory:')
|
|
19
|
+
elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier)
|
|
20
|
+
RequestLogAnalyzer::Database::Base.establish_connection(connection_hash)
|
|
21
|
+
elsif connection_identifier.kind_of?(String) # Normal SQLite 3 database file
|
|
22
|
+
RequestLogAnalyzer::Database::Base.establish_connection(:adapter => 'sqlite3', :database => connection_identifier)
|
|
23
|
+
elsif connection_identifier.nil?
|
|
24
|
+
nil
|
|
25
|
+
else
|
|
26
|
+
raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def disconnect
|
|
31
|
+
RequestLogAnalyzer::Database::Base.remove_connection
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def connection
|
|
35
|
+
RequestLogAnalyzer::Database::Base.connection
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
# Returns an array of all the Line objects of this request in the correct order.
|
|
4
|
+
def lines
|
|
5
|
+
@lines ||= begin
|
|
6
|
+
lines = []
|
|
7
|
+
self.class.reflections.each { |r, d| lines += self.send(r).all }
|
|
8
|
+
lines.sort
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Creates the table to store requests in.
|
|
13
|
+
def self.create_table!
|
|
14
|
+
unless database.connection.table_exists?(:requests)
|
|
15
|
+
database.connection.create_table(:requests) do |t|
|
|
16
|
+
t.column :first_lineno, :integer
|
|
17
|
+
t.column :last_lineno, :integer
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:sources)
|
|
5
|
+
database.connection.create_table(:sources) do |t|
|
|
6
|
+
t.column :filename, :string
|
|
7
|
+
t.column :mtime, :datetime
|
|
8
|
+
t.column :filesize, :integer
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
|
|
2
|
+
|
|
3
|
+
def self.create_table!
|
|
4
|
+
unless database.connection.table_exists?(:warnings)
|
|
5
|
+
database.connection.create_table(:warnings) do |t|
|
|
6
|
+
t.column :warning_type, :string, :limit => 30, :null => false
|
|
7
|
+
t.column :message, :string
|
|
8
|
+
t.column :source_id, :integer
|
|
9
|
+
t.column :lineno, :integer
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
end
|