request-log-analyzer 1.5.2 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. data/bin/request-log-analyzer +9 -5
  2. data/lib/request_log_analyzer.rb +1 -1
  3. data/lib/request_log_analyzer/aggregator/echo.rb +2 -1
  4. data/lib/request_log_analyzer/aggregator/summarizer.rb +5 -37
  5. data/lib/request_log_analyzer/controller.rb +4 -3
  6. data/lib/request_log_analyzer/file_format.rb +68 -1
  7. data/lib/request_log_analyzer/file_format/amazon_s3.rb +3 -1
  8. data/lib/request_log_analyzer/file_format/apache.rb +7 -2
  9. data/lib/request_log_analyzer/file_format/merb.rb +3 -1
  10. data/lib/request_log_analyzer/file_format/mysql.rb +10 -5
  11. data/lib/request_log_analyzer/file_format/rack.rb +1 -3
  12. data/lib/request_log_analyzer/file_format/rails.rb +5 -5
  13. data/lib/request_log_analyzer/output.rb +4 -0
  14. data/lib/request_log_analyzer/output/fancy_html.rb +31 -0
  15. data/lib/request_log_analyzer/tracker.rb +0 -105
  16. data/lib/request_log_analyzer/tracker/duration.rb +3 -62
  17. data/lib/request_log_analyzer/tracker/frequency.rb +18 -5
  18. data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
  19. data/lib/request_log_analyzer/tracker/traffic.rb +6 -72
  20. data/request-log-analyzer.gemspec +4 -4
  21. data/spec/lib/mocks.rb +1 -0
  22. data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
  23. data/spec/unit/file_format/mysql_format_spec.rb +1 -2
  24. data/spec/unit/file_format/rails_format_spec.rb +9 -4
  25. data/spec/unit/tracker/duration_tracker_spec.rb +2 -83
  26. data/spec/unit/tracker/{count_tracker_spec.rb → numeric_value_tracker_spec.rb} +50 -7
  27. data/spec/unit/tracker/tracker_api_spec.rb +3 -2
  28. data/spec/unit/tracker/traffic_tracker_spec.rb +0 -79
  29. metadata +8 -5
  30. data/lib/request_log_analyzer/tracker/count.rb +0 -93
@@ -19,54 +19,12 @@ module RequestLogAnalyzer::Tracker
19
19
  # EmployeeController#update.html [POST] | 4647 | 2731.23s | 0.59s
20
20
  # EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s
21
21
  # .............
22
- class Duration < Base
23
-
24
- include RequestLogAnalyzer::Tracker::StatisticsTracking
25
-
26
- attr_reader :categories
22
+ class Duration < NumericValue
27
23
 
28
24
  # Check if duration and catagory option have been received,
29
25
  def prepare
30
- raise "No duration field set up for category tracker #{self.inspect}" unless options[:duration]
31
- raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
32
-
33
- unless options[:multiple]
34
- @categorizer = create_lambda(options[:category])
35
- @durationizer = create_lambda(options[:duration])
36
- end
37
-
38
- @categories = {}
39
- end
40
-
41
- # Get the duration information fron the request and store it in the different categories.
42
- # <tt>request</tt> The request.
43
- def update(request)
44
- if options[:multiple]
45
- found_categories = request.every(options[:category])
46
- found_durations = request.every(options[:duration])
47
- raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_durations.length
48
- found_categories.each_with_index { |cat, index| update_statistics(cat, found_durations[index]) }
49
- else
50
- category = @categorizer.call(request)
51
- duration = @durationizer.call(request)
52
- update_statistics(category, duration) if duration.kind_of?(Numeric) && category
53
- end
54
- end
55
-
56
- # Block function to build a result table using a provided sorting function.
57
- # <tt>output</tt> The output object.
58
- # <tt>amount</tt> The number of rows in the report table (default 10).
59
- # === Options
60
- # * </tt>:title</tt> The title of the table
61
- # * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
62
- def report_table(output, sort, options = {}, &block)
63
- output.puts
64
- top_categories = output.slice_results(sorted_by(sort))
65
- output.with_style(:top_line => true) do
66
- output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
67
- top_categories.each { |(cat, info)| rows << statistics_row(cat) }
68
- end
69
- end
26
+ options[:value] = options[:duration] if options[:duration]
27
+ super
70
28
  end
71
29
 
72
30
  # Display a duration
@@ -79,26 +37,9 @@ module RequestLogAnalyzer::Tracker
79
37
  end
80
38
  end
81
39
 
82
- # Generate a request duration report to the given output object
83
- # By default colulative and average duration are generated.
84
- # Any options for the report should have been set during initialize.
85
- # <tt>output</tt> The output object
86
- def report(output)
87
- sortings = output.options[:sort] || [:sum, :mean]
88
- sortings.each do |sorting|
89
- report_table(output, sorting, :title => "#{title} - by #{sorting}")
90
- end
91
- end
92
-
93
40
  # Returns the title of this tracker for reports
94
41
  def title
95
42
  options[:title] || 'Request duration'
96
43
  end
97
-
98
- # Returns all the categories and the tracked duration as a hash than can be exported to YAML
99
- def to_yaml_object
100
- return nil if @categories.empty?
101
- @categories
102
- end
103
44
  end
104
45
  end
@@ -26,8 +26,10 @@ module RequestLogAnalyzer::Tracker
26
26
 
27
27
  # Check if categories are set up
28
28
  def prepare
29
+ options[:category] = options[:value] if options[:value] && !options[:category]
29
30
  raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
30
- @categorizer = create_lambda(options[:category])
31
+
32
+ @categorizer = create_lambda(options[:category]) unless options[:multiple]
31
33
 
32
34
  # Initialize the categories. Use the list of category names to
33
35
  @categories = {}
@@ -37,10 +39,21 @@ module RequestLogAnalyzer::Tracker
37
39
  # Check HTTP method of a request and store that in the categories hash.
38
40
  # <tt>request</tt> The request.
39
41
  def update(request)
40
- cat = @categorizer.call(request)
41
- if cat || options[:nils]
42
- @categories[cat] ||= 0
43
- @categories[cat] += 1
42
+ if options[:multiple]
43
+ cats = request.every(options[:category])
44
+ cats.each do |cat|
45
+ if cat || options[:nils]
46
+ @categories[cat] ||= 0
47
+ @categories[cat] += 1
48
+ end
49
+ end
50
+
51
+ else
52
+ cat = @categorizer.call(request)
53
+ if cat || options[:nils]
54
+ @categories[cat] ||= 0
55
+ @categories[cat] += 1
56
+ end
44
57
  end
45
58
  end
46
59
 
@@ -0,0 +1,223 @@
1
+ module RequestLogAnalyzer::Tracker
2
+
3
+ class NumericValue < Base
4
+
5
+ attr_reader :categories
6
+
7
+ # Sets up the numeric value tracker. It will check whether the value and category
8
+ # options are set that are used to extract and categorize the values during
9
+ # parsing. Two lambda procedures are created for these tasks
10
+ def prepare
11
+
12
+ raise "No value field set up for numeric tracker #{self.inspect}" unless options[:value]
13
+ raise "No categorizer set up for numeric tracker #{self.inspect}" unless options[:category]
14
+
15
+ unless options[:multiple]
16
+ @categorizer = create_lambda(options[:category])
17
+ @valueizer = create_lambda(options[:value])
18
+ end
19
+
20
+ @categories = {}
21
+ end
22
+
23
+ # Get the value information from the request and store it in the respective categories.
24
+ #
25
+ # If a request can contain multiple usable values for this tracker, the :multiple option
26
+ # should be set to true. In this case, all the values and respective categories will be
27
+ # read from the request using the #every method from the fields given in the :value and
28
+ # :category option.
29
+ #
30
+ # If the request contains only one suitable value and the :multiple is not set, it will
31
+ # read the single value and category from the fields provided in the :value and :category
32
+ # option, or calculate it with any lambda procedure that is assigned to these options. The
33
+ # request will be passed to procedure as input for the calculation.
34
+ #
35
+ # @param [RequestLogAnalyzer::Request] request The request to get the information from.
36
+ def update(request)
37
+ if options[:multiple]
38
+ found_categories = request.every(options[:category])
39
+ found_values = request.every(options[:value])
40
+ raise "Capture mismatch for multiple values in a request" unless found_categories.length == found_values.length
41
+
42
+ found_categories.each_with_index do |cat, index|
43
+ update_statistics(cat, found_values[index]) if cat && found_values[index].kind_of?(Numeric)
44
+ end
45
+
46
+ else
47
+ category = @categorizer.call(request)
48
+ value = @valueizer.call(request)
49
+ update_statistics(category, value) if value.kind_of?(Numeric) && category
50
+ end
51
+ end
52
+
53
+ # Block function to build a result table using a provided sorting function.
54
+ # <tt>output</tt> The output object.
55
+ # <tt>amount</tt> The number of rows in the report table (default 10).
56
+ # === Options
57
+ # * </tt>:title</tt> The title of the table
58
+ # * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
59
+ def report_table(output, sort, options = {}, &block)
60
+ output.puts
61
+ top_categories = output.slice_results(sorted_by(sort))
62
+ output.with_style(:top_line => true) do
63
+ output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
64
+ top_categories.each { |(cat, info)| rows << statistics_row(cat) }
65
+ end
66
+ end
67
+ end
68
+
69
+ # Display a value
70
+ def display_value(value)
71
+ return "- " if value.nil?
72
+ return "0 " if value.zero?
73
+
74
+ case Math.log10(value).floor
75
+ when 0...4 then '%d ' % value
76
+ when 4...7 then '%dk' % (value / 1000)
77
+ when 7...10 then '%dM' % (value / 1000_000)
78
+ when 10...13 then '%dG' % (value / 1000_000_000)
79
+ when 13...16 then '%dT' % (value / 1000_000_000_000)
80
+ else '%dP' % (value / 1000_000_000_000_000)
81
+ end
82
+ end
83
+
84
+ # Generate a request report to the given output object
85
+ # By default colulative and average duration are generated.
86
+ # Any options for the report should have been set during initialize.
87
+ # <tt>output</tt> The output object
88
+ def report(output)
89
+ sortings = output.options[:sort] || [:sum, :mean]
90
+ sortings.each do |sorting|
91
+ report_table(output, sorting, :title => "#{title} - by #{sorting}")
92
+ end
93
+
94
+ if options[:total]
95
+ output.puts
96
+ output.puts "#{output.colorize(title, :white, :bold)} - total: " + output.colorize(display_value(sum_overall), :brown, :bold)
97
+ end
98
+ end
99
+
100
+ # Returns the title of this tracker for reports
101
+ def title
102
+ @title ||= begin
103
+ if options[:title]
104
+ options[:title]
105
+ else
106
+ title_builder = ""
107
+ title_builder << "#{options[:value]} " if options[:value].kind_of?(Symbol)
108
+ title_builder << (options[:category].kind_of?(Symbol) ? "per #{options[:category]}" : "per request")
109
+ title_builder
110
+ end
111
+ end
112
+ end
113
+
114
+ # Returns all the categories and the tracked duration as a hash than can be exported to YAML
115
+ def to_yaml_object
116
+ return nil if @categories.empty?
117
+ @categories
118
+ end
119
+
120
+
121
+ # Update sthe running calculation of statistics with the newly found numeric value.
122
+ # <tt>category</tt>:: The category for which to update the running statistics calculations
123
+ # <tt>number</tt>:: The numeric value to update the calculations with.
124
+ def update_statistics(category, number)
125
+ @categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
126
+ delta = number - @categories[category][:mean]
127
+
128
+ @categories[category][:hits] += 1
129
+ @categories[category][:mean] += (delta / @categories[category][:hits])
130
+ @categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
131
+ @categories[category][:sum] += number
132
+ @categories[category][:min] = number if number < @categories[category][:min]
133
+ @categories[category][:max] = number if number > @categories[category][:max]
134
+ end
135
+
136
+ # Get the number of hits of a specific category.
137
+ # <tt>cat</tt> The category
138
+ def hits(cat)
139
+ @categories[cat][:hits]
140
+ end
141
+
142
+ # Get the total duration of a specific category.
143
+ # <tt>cat</tt> The category
144
+ def sum(cat)
145
+ @categories[cat][:sum]
146
+ end
147
+
148
+ # Get the minimal duration of a specific category.
149
+ # <tt>cat</tt> The category
150
+ def min(cat)
151
+ @categories[cat][:min]
152
+ end
153
+
154
+ # Get the maximum duration of a specific category.
155
+ # <tt>cat</tt> The category
156
+ def max(cat)
157
+ @categories[cat][:max]
158
+ end
159
+
160
+ # Get the average duration of a specific category.
161
+ # <tt>cat</tt> The category
162
+ def mean(cat)
163
+ @categories[cat][:mean]
164
+ end
165
+
166
+ # Get the standard deviation of the duration of a specific category.
167
+ # <tt>cat</tt> The category
168
+ def stddev(cat)
169
+ Math.sqrt(variance(cat))
170
+ end
171
+
172
+ # Get the variance of the duration of a specific category.
173
+ # <tt>cat</tt> The category
174
+ def variance(cat)
175
+ return 0.0 if @categories[cat][:hits] <= 1
176
+ (@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
177
+ end
178
+
179
+ # Get the average duration of a all categories.
180
+ def mean_overall
181
+ sum_overall / hits_overall
182
+ end
183
+
184
+ # Get the cumlative duration of a all categories.
185
+ def sum_overall
186
+ @categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
187
+ end
188
+
189
+ # Get the total hits of a all categories.
190
+ def hits_overall
191
+ @categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
192
+ end
193
+
194
+ # Return categories sorted by a given key.
195
+ # <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
196
+ def sorted_by(by = nil)
197
+ if block_given?
198
+ categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
199
+ else
200
+ categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
201
+ end
202
+ end
203
+
204
+ # Returns the column header for a statistics table to report on the statistics result
205
+ def statistics_header(options)
206
+ [
207
+ {:title => options[:title], :width => :rest},
208
+ {:title => 'Hits', :align => :right, :highlight => (options[:highlight] == :hits), :min_width => 4},
209
+ {:title => 'Sum', :align => :right, :highlight => (options[:highlight] == :sum), :min_width => 6},
210
+ {:title => 'Mean', :align => :right, :highlight => (options[:highlight] == :mean), :min_width => 6},
211
+ {:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
212
+ {:title => 'Min', :align => :right, :highlight => (options[:highlight] == :min), :min_width => 6},
213
+ {:title => 'Max', :align => :right, :highlight => (options[:highlight] == :max), :min_width => 6}
214
+ ]
215
+ end
216
+
217
+ # Returns a row of statistics information for a report table, given a category
218
+ def statistics_row(cat)
219
+ [cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
220
+ display_value(min(cat)), display_value(max(cat))]
221
+ end
222
+ end
223
+ end
@@ -9,55 +9,22 @@ module RequestLogAnalyzer::Tracker
9
9
  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
10
10
  # * <tt>:title</tt> Title do be displayed above the report
11
11
  # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
12
- class Traffic < Base
13
-
14
- attr_reader :categories
15
-
16
- include RequestLogAnalyzer::Tracker::StatisticsTracking
12
+ class Traffic < NumericValue
17
13
 
18
14
  # Check if duration and catagory option have been received,
19
15
  def prepare
20
- raise "No traffic field set up for category tracker #{self.inspect}" unless options[:traffic]
21
- raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
22
-
23
- @categorizer = create_lambda(options[:category])
24
- @trafficizer = create_lambda(options[:traffic])
25
- @categories = {}
26
- end
27
-
28
-
29
- # Get the duration information from the request and store it in the different categories.
30
- # <tt>request</tt> The request.
31
- def update(request)
32
- category = @categorizer.call(request)
33
- traffic = @trafficizer.call(request)
34
- update_statistics(category, traffic) if traffic.kind_of?(Numeric) && !category.nil?
35
- end
36
-
37
- # Block function to build a result table using a provided sorting function.
38
- # <tt>output</tt> The output object.
39
- # <tt>amount</tt> The number of rows in the report table (default 10).
40
- # === Options
41
- # * </tt>:title</tt> The title of the table
42
- # * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
43
- def report_table(output, sort, options = {}, &block)
44
- output.puts
45
-
46
- top_categories = output.slice_results(sorted_by(sort))
47
- output.with_style(:top_line => true) do
48
- output.table(*statistics_header(:title => options[:title],:highlight => sort)) do |rows|
49
- top_categories.each { |(cat, info)| rows.push(statistics_row(cat)) }
50
- end
51
- end
52
- output.puts
16
+ options[:value] = options[:traffic] if options[:traffic]
17
+ options[:total] = true
18
+ super
53
19
  end
54
20
 
55
21
  # Formats the traffic number using x B/kB/MB/GB etc notation
56
22
  def display_value(bytes)
57
23
  return "-" if bytes.nil?
58
24
  return "0 B" if bytes.zero?
25
+
59
26
  case Math.log10(bytes).floor
60
- when 1...4 then '%d B' % bytes
27
+ when 0...4 then '%d B' % bytes
61
28
  when 4...7 then '%d kB' % (bytes / 1000)
62
29
  when 7...10 then '%d MB' % (bytes / 1000_000)
63
30
  when 10...13 then '%d GB' % (bytes / 1000_000_000)
@@ -65,42 +32,9 @@ module RequestLogAnalyzer::Tracker
65
32
  end
66
33
  end
67
34
 
68
- # Generate a request duration report to the given output object
69
- # By default colulative and average duration are generated.
70
- # Any options for the report should have been set during initialize.
71
- # <tt>output</tt> The output object
72
- def report(output)
73
-
74
- sortings = output.options[:sort] || [:sum, :mean]
75
-
76
- sortings.each do |report|
77
- case report
78
- when :mean
79
- report_table(output, :mean, :title => "#{title} - sorted by mean")
80
- when :stddev
81
- report_table(output, :stddev, :title => "#{title} - sorted by standard deviation")
82
- when :sum
83
- report_table(output, :sum, :title => "#{title} - sorted by sum")
84
- when :hits
85
- report_table(output, :hits, :title => "#{title} - sorted by hits")
86
- else
87
- raise "Unknown duration report specified: #{report}!"
88
- end
89
- end
90
-
91
- output.puts
92
- output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(display_value(sum_overall), :brown, :bold)
93
- end
94
-
95
35
  # Returns the title of this tracker for reports
96
36
  def title
97
37
  options[:title] || 'Request traffic'
98
38
  end
99
-
100
- # Returns all the categories and the tracked duration as a hash than can be exported to YAML
101
- def to_yaml_object
102
- return nil if @categories.empty?
103
- @categories
104
- end
105
39
  end
106
40
  end