request-log-analyzer 1.0.2 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +10 -0
- data/DESIGN.rdoc +41 -0
- data/LICENSE +4 -4
- data/README.rdoc +38 -0
- data/Rakefile +6 -3
- data/bin/request-log-analyzer +70 -72
- data/lib/cli/command_line_arguments.rb +53 -53
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +43 -0
- data/lib/cli/progressbar.rb +166 -189
- data/lib/cli/tools.rb +49 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
- data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
- data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
- data/lib/request_log_analyzer/controller.rb +251 -98
- data/lib/request_log_analyzer/database/base.rb +114 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +102 -0
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
- data/lib/request_log_analyzer/file_format/apache.rb +147 -0
- data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
- data/lib/request_log_analyzer/file_format/merb.rb +65 -29
- data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
- data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
- data/lib/request_log_analyzer/file_format/rack.rb +9 -0
- data/lib/request_log_analyzer/file_format/rails.rb +164 -78
- data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
- data/lib/request_log_analyzer/file_format.rb +252 -58
- data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
- data/lib/request_log_analyzer/filter/field.rb +19 -13
- data/lib/request_log_analyzer/filter/timespan.rb +25 -12
- data/lib/request_log_analyzer/filter.rb +30 -0
- data/lib/request_log_analyzer/line_definition.rb +69 -96
- data/lib/request_log_analyzer/log_processor.rb +31 -53
- data/lib/request_log_analyzer/mailer.rb +65 -0
- data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
- data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
- data/lib/request_log_analyzer/output/html.rb +187 -0
- data/lib/request_log_analyzer/output.rb +117 -0
- data/lib/request_log_analyzer/request.rb +125 -40
- data/lib/request_log_analyzer/source/database_loader.rb +87 -0
- data/lib/request_log_analyzer/source/log_parser.rb +297 -0
- data/lib/request_log_analyzer/source.rb +72 -0
- data/lib/request_log_analyzer/tracker/duration.rb +28 -64
- data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
- data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
- data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
- data/lib/request_log_analyzer/tracker.rb +101 -0
- data/lib/request_log_analyzer.rb +43 -13
- data/request-log-analyzer.gemspec +41 -0
- data/spec/database.yml +23 -0
- data/spec/fixtures/apache_combined.log +5 -0
- data/spec/fixtures/apache_common.log +10 -0
- data/spec/fixtures/decompression.log +12 -0
- data/spec/fixtures/decompression.log.bz2 +0 -0
- data/spec/fixtures/decompression.log.gz +0 -0
- data/spec/fixtures/decompression.log.zip +0 -0
- data/spec/fixtures/decompression.tar.gz +0 -0
- data/spec/fixtures/decompression.tgz +0 -0
- data/spec/fixtures/header_and_footer.log +6 -0
- data/spec/fixtures/merb_prefixed.log +9 -0
- data/spec/fixtures/mysql_slow_query.log +110 -0
- data/spec/fixtures/postgresql.log +2980 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/fixtures/sinatra.log +99 -0
- data/spec/integration/command_line_usage_spec.rb +84 -0
- data/spec/integration/mailer_spec.rb +179 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +152 -0
- data/spec/lib/helpers.rb +72 -0
- data/spec/lib/macros.rb +18 -0
- data/spec/lib/matchers.rb +77 -0
- data/spec/lib/mocks.rb +77 -0
- data/spec/lib/testing_format.rb +46 -0
- data/spec/spec_helper.rb +16 -59
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/aggregator/summarizer_spec.rb +26 -0
- data/spec/unit/controller/controller_spec.rb +41 -0
- data/spec/unit/controller/log_processor_spec.rb +18 -0
- data/spec/unit/database/base_class_spec.rb +183 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +133 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
- data/spec/unit/file_format/apache_format_spec.rb +203 -0
- data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
- data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
- data/spec/unit/file_format/file_format_api_spec.rb +69 -0
- data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
- data/spec/unit/file_format/line_definition_spec.rb +75 -0
- data/spec/unit/file_format/merb_format_spec.rb +52 -0
- data/spec/unit/file_format/mysql_format_spec.rb +154 -0
- data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
- data/spec/unit/file_format/rack_format_spec.rb +50 -0
- data/spec/unit/file_format/rails3_format_spec.rb +120 -0
- data/spec/unit/file_format/rails_format_spec.rb +180 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
- data/spec/unit/filter/field_filter_spec.rb +66 -0
- data/spec/unit/filter/filter_spec.rb +17 -0
- data/spec/unit/filter/timespan_filter_spec.rb +58 -0
- data/spec/unit/mailer_spec.rb +42 -0
- data/spec/unit/request_spec.rb +111 -0
- data/spec/unit/source/log_parser_spec.rb +119 -0
- data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
- data/spec/unit/tracker/tracker_api_spec.rb +125 -0
- data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
- data/tasks/github-gem.rake +290 -139
- data/tasks/request_log_analyzer.rake +23 -7
- metadata +221 -94
- data/DESIGN +0 -14
- data/HACKING +0 -7
- data/README.textile +0 -36
- data/lib/cli/bashcolorizer.rb +0 -60
- data/lib/request_log_analyzer/aggregator/database.rb +0 -148
- data/lib/request_log_analyzer/filter/base.rb +0 -29
- data/lib/request_log_analyzer/log_parser.rb +0 -173
- data/lib/request_log_analyzer/source/base.rb +0 -42
- data/lib/request_log_analyzer/source/log_file.rb +0 -170
- data/lib/request_log_analyzer/tracker/base.rb +0 -54
- data/lib/request_log_analyzer/tracker/category.rb +0 -71
- data/spec/controller_spec.rb +0 -40
- data/spec/database_inserter_spec.rb +0 -101
- data/spec/file_format_spec.rb +0 -78
- data/spec/file_formats/spec_format.rb +0 -26
- data/spec/filter_spec.rb +0 -137
- data/spec/line_definition_spec.rb +0 -124
- data/spec/log_parser_spec.rb +0 -68
- data/spec/log_processor_spec.rb +0 -57
- data/spec/merb_format_spec.rb +0 -38
- data/spec/rails_format_spec.rb +0 -76
- data/spec/request_spec.rb +0 -72
- data/spec/summarizer_spec.rb +0 -9
- data/tasks/rspec.rake +0 -6
|
@@ -1,113 +1,151 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/../tracker/base'
|
|
2
|
-
|
|
3
1
|
module RequestLogAnalyzer::Aggregator
|
|
4
2
|
|
|
5
3
|
class Summarizer < Base
|
|
6
|
-
|
|
4
|
+
|
|
7
5
|
class Definer
|
|
8
|
-
|
|
6
|
+
|
|
9
7
|
attr_reader :trackers
|
|
10
|
-
|
|
8
|
+
|
|
9
|
+
# Initialize tracker array
|
|
11
10
|
def initialize
|
|
12
11
|
@trackers = []
|
|
13
12
|
end
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
|
|
14
|
+
# Initialize tracker summarizer by duping the trackers of another summarizer
|
|
15
|
+
# <tt>other</tt> The other Summarizer
|
|
16
|
+
def initialize_copy(other)
|
|
17
|
+
@trackers = other.trackers.dup
|
|
17
18
|
end
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
19
|
+
|
|
20
|
+
# Drop all trackers
|
|
21
|
+
def reset!
|
|
22
|
+
@trackers = []
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Include missing trackers through method missing.
|
|
26
|
+
def method_missing(tracker_method, *args)
|
|
27
|
+
track(tracker_method, *args)
|
|
25
28
|
end
|
|
26
29
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
def track(tracker_klass, options = {})
|
|
36
|
-
require "#{File.dirname(__FILE__)}/../tracker/#{tracker_klass}"
|
|
37
|
-
tracker_klass = RequestLogAnalyzer::Tracker.const_get(tracker_klass.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) if tracker_klass.kind_of?(Symbol)
|
|
30
|
+
# Helper function to initialize a tracker and add it to the tracker array.
|
|
31
|
+
# <tt>tracker_class</tt> The class to include
|
|
32
|
+
# <tt>optiont</tt> The options to pass to the trackers.
|
|
33
|
+
def track(tracker_klass, value_field = {}, other_options = {})
|
|
34
|
+
options = value_field.kind_of?(Symbol) ? other_options.merge(:value => value_field) : value_field.merge(other_options)
|
|
35
|
+
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer::to_camelcase(tracker_klass)) if tracker_klass.kind_of?(Symbol)
|
|
38
36
|
@trackers << tracker_klass.new(options)
|
|
39
37
|
end
|
|
40
38
|
end
|
|
41
|
-
|
|
39
|
+
|
|
42
40
|
attr_reader :trackers
|
|
43
41
|
attr_reader :warnings_encountered
|
|
44
|
-
|
|
42
|
+
|
|
43
|
+
# Initialize summarizer.
|
|
44
|
+
# Generate trackers from speciefied source.file_format.report_trackers and set them up
|
|
45
45
|
def initialize(source, options = {})
|
|
46
46
|
super(source, options)
|
|
47
47
|
@warnings_encountered = {}
|
|
48
48
|
@trackers = source.file_format.report_trackers
|
|
49
49
|
setup
|
|
50
50
|
end
|
|
51
|
-
|
|
51
|
+
|
|
52
52
|
def setup
|
|
53
53
|
end
|
|
54
|
-
|
|
54
|
+
|
|
55
|
+
# Call prepare on all trackers.
|
|
55
56
|
def prepare
|
|
56
57
|
raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
|
|
57
58
|
@trackers.each { |tracker| tracker.prepare }
|
|
58
|
-
|
|
59
|
-
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Pass all requests to trackers and let them update if necessary.
|
|
62
|
+
# <tt>request</tt> The request to pass.
|
|
60
63
|
def aggregate(request)
|
|
61
64
|
@trackers.each do |tracker|
|
|
62
65
|
tracker.update(request) if tracker.should_update?(request)
|
|
63
66
|
end
|
|
64
67
|
end
|
|
65
|
-
|
|
68
|
+
|
|
69
|
+
# Call finalize on all trackers. Saves a YAML dump if this is set in the options.
|
|
66
70
|
def finalize
|
|
67
71
|
@trackers.each { |tracker| tracker.finalize }
|
|
72
|
+
save_results_dump(options[:yaml]) if options[:yaml]
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Saves the results of all the trackers in YAML format to a file.
|
|
76
|
+
# <tt>filename</tt> The file to store the YAML dump in.
|
|
77
|
+
def save_results_dump(filename)
|
|
78
|
+
File.open(filename, 'w') { |file| file.write(to_yaml) }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Exports all the tracker results to YAML. It will call the to_yaml_object method
|
|
82
|
+
# for every tracker and combines these into a single YAML export.
|
|
83
|
+
def to_yaml
|
|
84
|
+
require 'yaml'
|
|
85
|
+
trackers_export = @trackers.inject({}) do |export, tracker|
|
|
86
|
+
export[tracker.title] = tracker.to_yaml_object; export
|
|
87
|
+
end
|
|
88
|
+
YAML::dump(trackers_export)
|
|
68
89
|
end
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
90
|
+
|
|
91
|
+
# Call report on all trackers.
|
|
92
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
|
93
|
+
def report(output)
|
|
94
|
+
report_header(output)
|
|
72
95
|
if source.parsed_requests > 0
|
|
73
|
-
@trackers.each { |tracker|
|
|
96
|
+
@trackers.each { |tracker| output.report_tracker(tracker) }
|
|
74
97
|
else
|
|
75
|
-
output
|
|
76
|
-
output
|
|
98
|
+
output.puts
|
|
99
|
+
output.puts('There were no requests analyzed.')
|
|
77
100
|
end
|
|
78
|
-
report_footer(output
|
|
101
|
+
report_footer(output)
|
|
79
102
|
end
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
output
|
|
85
|
-
|
|
86
|
-
output
|
|
87
|
-
|
|
88
|
-
|
|
103
|
+
|
|
104
|
+
# Generate report header.
|
|
105
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
|
106
|
+
def report_header(output)
|
|
107
|
+
output.title("Request summary")
|
|
108
|
+
|
|
109
|
+
output.with_style(:cell_separator => false) do
|
|
110
|
+
output.table({:width => 20}, {:font => :bold}) do |rows|
|
|
111
|
+
rows << ['Parsed lines:', source.parsed_lines]
|
|
112
|
+
rows << ['Skipped lines:', source.skipped_lines]
|
|
113
|
+
rows << ['Parsed requests:', source.parsed_requests]
|
|
114
|
+
rows << ['Skipped requests:', source.skipped_requests]
|
|
115
|
+
rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
|
|
116
|
+
end
|
|
89
117
|
end
|
|
90
118
|
output << "\n"
|
|
91
119
|
end
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
output
|
|
98
|
-
|
|
99
|
-
output
|
|
120
|
+
|
|
121
|
+
# Generate report footer.
|
|
122
|
+
# <tt>output</tt> RequestLogAnalyzer::Output object to output to
|
|
123
|
+
def report_footer(output)
|
|
124
|
+
if has_log_ordering_warnings?
|
|
125
|
+
output.title("Parse warnings")
|
|
126
|
+
|
|
127
|
+
output.puts "Parsable lines were encountered without a header line before it. It"
|
|
128
|
+
output.puts "could be that logging is not setup correctly for your application."
|
|
129
|
+
output.puts "Visit this website for logging configuration tips:"
|
|
130
|
+
output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
|
|
131
|
+
output.puts
|
|
100
132
|
end
|
|
101
133
|
end
|
|
102
|
-
|
|
134
|
+
|
|
135
|
+
# Returns true if there were any warnings generated by the trackers
|
|
103
136
|
def has_warnings?
|
|
104
|
-
|
|
137
|
+
@warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
|
|
105
138
|
end
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
139
|
+
|
|
140
|
+
# Returns true if there were any log ordering warnings
|
|
141
|
+
def has_log_ordering_warnings?
|
|
142
|
+
@warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
|
|
109
143
|
end
|
|
110
|
-
|
|
144
|
+
|
|
145
|
+
# Store an encountered warning
|
|
146
|
+
# <tt>type</tt> Type of warning
|
|
147
|
+
# <tt>message</tt> Warning message
|
|
148
|
+
# <tt>lineno</tt> The line on which the error was encountered
|
|
111
149
|
def warning(type, message, lineno)
|
|
112
150
|
@warnings_encountered[type] ||= 0
|
|
113
151
|
@warnings_encountered[type] += 1
|
|
@@ -1,44 +1,48 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
|
2
2
|
|
|
3
|
+
def self.const_missing(const)
|
|
4
|
+
RequestLogAnalyzer::load_default_class_file(self, const)
|
|
5
|
+
end
|
|
6
|
+
|
|
3
7
|
# The base class of an aggregator. This class provides the interface to which
|
|
4
8
|
# every aggregator should comply (by simply subclassing this class).
|
|
5
9
|
class Base
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
attr_reader :options
|
|
10
|
-
attr_reader :source
|
|
10
|
+
|
|
11
|
+
attr_reader :options, :source
|
|
11
12
|
|
|
12
13
|
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
|
13
14
|
# It will include the specific file format module.
|
|
14
15
|
def initialize(source, options = {})
|
|
15
16
|
@source = source
|
|
16
|
-
self.register_file_format(source.file_format)
|
|
17
17
|
@options = options
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
# The prepare function is called just before parsing starts. This function
|
|
20
|
+
# The prepare function is called just before parsing starts. This function
|
|
21
21
|
# can be used to initialie variables, etc.
|
|
22
22
|
def prepare
|
|
23
23
|
end
|
|
24
|
-
|
|
24
|
+
|
|
25
25
|
# The aggregate function is called for every request.
|
|
26
26
|
# Implement the aggregating functionality in this method
|
|
27
27
|
def aggregate(request)
|
|
28
28
|
end
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
# The finalize function is called after all sources are parsed and no more
|
|
31
31
|
# requests will be passed to the aggregator
|
|
32
32
|
def finalize
|
|
33
33
|
end
|
|
34
|
-
|
|
34
|
+
|
|
35
35
|
# The warning method is called if the parser eits a warning.
|
|
36
36
|
def warning(type, message, lineno)
|
|
37
|
-
end
|
|
38
|
-
|
|
37
|
+
end
|
|
38
|
+
|
|
39
39
|
# The report function is called at the end. Implement any result reporting
|
|
40
40
|
# in this function.
|
|
41
|
-
def report(output
|
|
41
|
+
def report(output)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The source_change function gets called when handling a source is started or finished.
|
|
45
|
+
def source_change(change, filename)
|
|
42
46
|
end
|
|
43
47
|
|
|
44
48
|
end
|