request-log-analyzer 1.3.7 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +2 -2
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer.rb +4 -4
  10. data/lib/request_log_analyzer/aggregator.rb +10 -10
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
  12. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  14. data/lib/request_log_analyzer/controller.rb +153 -69
  15. data/lib/request_log_analyzer/database.rb +13 -13
  16. data/lib/request_log_analyzer/database/base.rb +17 -17
  17. data/lib/request_log_analyzer/database/connection.rb +3 -3
  18. data/lib/request_log_analyzer/database/request.rb +2 -2
  19. data/lib/request_log_analyzer/database/source.rb +1 -1
  20. data/lib/request_log_analyzer/file_format.rb +15 -15
  21. data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
  22. data/lib/request_log_analyzer/file_format/apache.rb +20 -19
  23. data/lib/request_log_analyzer/file_format/merb.rb +12 -12
  24. data/lib/request_log_analyzer/file_format/rack.rb +4 -4
  25. data/lib/request_log_analyzer/file_format/rails.rb +146 -70
  26. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  27. data/lib/request_log_analyzer/filter.rb +6 -6
  28. data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
  29. data/lib/request_log_analyzer/filter/field.rb +9 -9
  30. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +22 -22
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output.rb +53 -12
  35. data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
  36. data/lib/request_log_analyzer/output/html.rb +20 -20
  37. data/lib/request_log_analyzer/request.rb +35 -36
  38. data/lib/request_log_analyzer/source.rb +7 -7
  39. data/lib/request_log_analyzer/source/database_loader.rb +7 -7
  40. data/lib/request_log_analyzer/source/log_parser.rb +48 -43
  41. data/lib/request_log_analyzer/tracker.rb +128 -14
  42. data/lib/request_log_analyzer/tracker/duration.rb +39 -132
  43. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  44. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
  45. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  46. data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
  47. data/request-log-analyzer.gemspec +19 -15
  48. data/spec/fixtures/rails_22.log +1 -1
  49. data/spec/integration/command_line_usage_spec.rb +1 -1
  50. data/spec/lib/helpers.rb +7 -7
  51. data/spec/lib/macros.rb +3 -3
  52. data/spec/lib/matchers.rb +41 -27
  53. data/spec/lib/mocks.rb +15 -14
  54. data/spec/lib/testing_format.rb +9 -9
  55. data/spec/spec_helper.rb +6 -6
  56. data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
  57. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  58. data/spec/unit/controller/controller_spec.rb +2 -2
  59. data/spec/unit/controller/log_processor_spec.rb +1 -1
  60. data/spec/unit/database/base_class_spec.rb +19 -19
  61. data/spec/unit/database/connection_spec.rb +3 -3
  62. data/spec/unit/database/database_spec.rb +25 -25
  63. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  64. data/spec/unit/file_format/apache_format_spec.rb +13 -13
  65. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  66. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  67. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  68. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  69. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  70. data/spec/unit/filter/field_filter_spec.rb +13 -13
  71. data/spec/unit/filter/filter_spec.rb +1 -1
  72. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  73. data/spec/unit/mailer_spec.rb +30 -0
  74. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  75. data/spec/unit/source/log_parser_spec.rb +27 -27
  76. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  77. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  78. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  79. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  80. data/spec/unit/tracker/tracker_api_spec.rb +13 -13
  81. data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
  82. data/tasks/github-gem.rake +125 -75
  83. data/tasks/request_log_analyzer.rake +2 -2
  84. metadata +8 -6
data/LICENSE CHANGED
@@ -1,5 +1,5 @@
1
1
  Copyright (c) 2008 Willem van Bergen / Bart ten Brinke
2
-
2
+
3
3
  Permission is hereby granted, free of charge, to any person obtaining
4
4
  a copy of this software and associated documentation files (the
5
5
  "Software"), to deal in the Software without restriction, including
@@ -7,10 +7,10 @@ without limitation the rights to use, copy, modify, merge, publish,
7
7
  distribute, sublicense, and/or sell copies of the Software, and to
8
8
  permit persons to whom the Software is furnished to do so, subject to
9
9
  the following conditions:
10
-
10
+
11
11
  The above copyright notice and this permission notice shall be
12
12
  included in all copies or substantial portions of the Software.
13
-
13
+
14
14
  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
15
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
16
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
data/README.rdoc CHANGED
@@ -21,7 +21,7 @@ Alternatively, use the gem from the GitHub gem server:
21
21
 
22
22
  $ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
23
23
 
24
- To get the best results out of request-log-analyzer, make sure to
24
+ To get the best results out of request-log-analyzer, make sure to
25
25
  set up logging correctly: http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
26
26
  for your application.
27
27
 
@@ -1,4 +1,6 @@
1
- #!/usr/bin/ruby
1
+ #!/usr/bin/env ruby
2
+ # encoding: utf-8
3
+
2
4
  $:.unshift(File.dirname(__FILE__) + '/../lib')
3
5
  require 'request_log_analyzer'
4
6
  require 'cli/command_line_arguments'
@@ -15,8 +17,6 @@ begin
15
17
 
16
18
  command_line.command(:console) do |cons|
17
19
  cons.option(:database, :alias => :d, :required => true)
18
- cons.option(:format, :alias => :f, :default => 'rails')
19
- cons.option(:apache_format)
20
20
  end
21
21
 
22
22
  command_line.command(:strip) do |strip|
@@ -24,11 +24,12 @@ begin
24
24
  strip.option(:format, :alias => :f, :default => 'rails')
25
25
  strip.option(:output, :alias => :o)
26
26
  strip.switch(:discard_teaser_lines, :t)
27
- strip.switch(:keep_junk_lines, :j)
27
+ strip.switch(:keep_junk_lines, :j)
28
28
  end
29
29
 
30
30
  command_line.option(:format, :alias => :f, :default => 'rails')
31
31
  command_line.option(:apache_format)
32
+ command_line.option(:rails_format)
32
33
 
33
34
  command_line.option(:file, :alias => :e)
34
35
  command_line.option(:mail, :alias => :m)
@@ -48,7 +49,9 @@ begin
48
49
 
49
50
  command_line.switch(:boring, :b)
50
51
  command_line.option(:output, :alias => :o, :default => 'FixedWidth')
51
- command_line.option(:report_width, :default => terminal_width - 1)
52
+ command_line.option(:report_width, :default => terminal_width - 1)
53
+ command_line.option(:report_amount, :default => 20)
54
+ command_line.option(:report_sort, :default => 'sum,mean')
52
55
 
53
56
  command_line.switch(:debug)
54
57
  command_line.switch(:no_progress)
@@ -57,7 +60,7 @@ begin
57
60
  end
58
61
 
59
62
  rescue CommandLine::Error => e
60
- puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
63
+ puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
61
64
  puts "Website: http://railsdoctors.com"
62
65
  puts
63
66
  puts "ARGUMENT ERROR: " + e.message if e.message
@@ -67,9 +70,9 @@ rescue CommandLine::Error => e
67
70
  puts "Input options:"
68
71
  puts " --format <format>, -f: Uses the specified log file format. Defaults to rails."
69
72
  puts " --after <date> Only consider requests from <date> or later."
70
- puts " --before <date> Only consider requests before <date>."
71
- puts " --select <field> <value> Only consider requests where <field> matches <value>."
72
- puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
73
+ puts " --before <date> Only consider requests before <date>."
74
+ puts " --select <field> <value> Only consider requests where <field> matches <value>."
75
+ puts " --reject <field> <value> Only consider requests where <field> does not match <value>."
73
76
  puts
74
77
  puts "Output options:"
75
78
  puts " --boring, -b Output reports without ASCII colors."
@@ -77,9 +80,9 @@ rescue CommandLine::Error => e
77
80
  puts " --debug Print debug information while parsing."
78
81
  puts " --file <filename> Output to file."
79
82
  puts " --mail <emailaddress> Send report to an email address."
80
- puts " --output <format> Output format. Supports 'HTML' and 'FixedWidth' (default)"
83
+ puts " --output <format> Output format. Supports 'HTML' and 'FixedWidth' (default)"
81
84
  puts " --dump <filename> Dump the YAML formatted results in the given file"
82
- puts
85
+ puts
83
86
  puts "Examples:"
84
87
  puts " request-log-analyzer development.log"
85
88
  puts " request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
@@ -89,7 +92,7 @@ rescue CommandLine::Error => e
89
92
  puts "run the following command in your application's root directory:"
90
93
  puts
91
94
  puts " request-log-analyzer install rails"
92
- exit(0)
95
+ exit(0)
93
96
  end
94
97
 
95
98
  case arguments.command
@@ -101,11 +104,11 @@ when :console
101
104
  when :strip
102
105
  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'
103
106
  RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
104
- else
107
+ else
105
108
  puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"
106
109
  puts "Website: http://railsdoctors.com"
107
110
  puts
108
111
 
109
112
  # Run the request_log_analyzer!
110
- RequestLogAnalyzer::Controller.build(arguments).run!
113
+ RequestLogAnalyzer::Controller.build_from_arguments(arguments).run!
111
114
  end
@@ -71,7 +71,7 @@ module CommandLine
71
71
  def required?
72
72
  @required
73
73
  end
74
-
74
+
75
75
  # Check if flag is optional
76
76
  def optional?
77
77
  !@required
@@ -85,22 +85,22 @@ module CommandLine
85
85
  !@default_value.nil?
86
86
  end
87
87
  end
88
-
88
+
89
89
  class Arguments
90
90
 
91
91
  class Definition
92
-
92
+
93
93
  ENDLESS_PARAMETERS = 99999
94
-
94
+
95
95
  attr_reader :commands, :options, :parameters
96
-
96
+
97
97
  def initialize(parent)
98
98
  @parent = parent
99
99
  @options = {}
100
100
  @commands = {}
101
101
  @parameters = nil
102
102
  end
103
-
103
+
104
104
  def [](option_name)
105
105
  option_symbol = CommandLine::Option.rewrite(option_name)
106
106
  if the_option = @options.detect { |(name, odef)| odef =~ option_symbol }
@@ -109,22 +109,22 @@ module CommandLine
109
109
  raise CommandLine::UnknownOption, option_name
110
110
  end
111
111
  end
112
-
112
+
113
113
  def minimum_parameters=(count_specifier)
114
114
  @parameters = count_specifier..ENDLESS_PARAMETERS
115
115
  end
116
-
116
+
117
117
  def parameters=(count_specifier)
118
118
  @parameters = count_specifier
119
119
  end
120
-
121
- alias :files= :parameters=
122
-
120
+
121
+ alias :files= :parameters=
122
+
123
123
  def option(name, options = {})
124
124
  clo = CommandLine::Option.new(name, options)
125
125
  @options[clo.name] = clo
126
126
  end
127
-
127
+
128
128
  def switch(name, switch_alias = nil)
129
129
  option(name, :alias => switch_alias, :parameters => 0)
130
130
  end
@@ -134,35 +134,35 @@ module CommandLine
134
134
  yield(command_definition) if block_given?
135
135
  @commands[CommandLine::Option.rewrite(name)] = command_definition
136
136
  end
137
-
137
+
138
138
  def has_command?(command)
139
139
  @commands[CommandLine::Option.rewrite(command)]
140
140
  end
141
141
  end
142
-
142
+
143
143
  OPTION_REGEXP = /^\-\-([A-Za-z0-9-]+)$/;
144
144
  ALIASES_REGEXP = /^\-([A-Aa-z0-9]+)$/
145
-
145
+
146
146
  attr_reader :definition
147
147
  attr_reader :tokens
148
148
  attr_reader :command, :options, :parameters
149
-
149
+
150
150
  def self.parse(tokens = $*, &block)
151
151
  cla = Arguments.new
152
152
  cla.define(&block)
153
- return cla.parse!(tokens)
153
+ return cla.parse!(tokens)
154
154
  end
155
-
155
+
156
156
  def initialize
157
157
  @tokens = []
158
- @definition = Definition.new(self)
159
- @current_definition = @definition
158
+ @definition = Definition.new(self)
159
+ @current_definition = @definition
160
160
  end
161
-
161
+
162
162
  def define(&block)
163
163
  yield(@definition)
164
164
  end
165
-
165
+
166
166
  def [](option)
167
167
  if the_option = @options.detect { |(key, value)| key =~ option }
168
168
  the_option[1]
@@ -170,31 +170,31 @@ module CommandLine
170
170
  @current_definition[option].default_value
171
171
  end
172
172
  end
173
-
173
+
174
174
  def next_token
175
175
  @current_token = @tokens.shift
176
176
  return @current_token
177
177
  end
178
-
178
+
179
179
  def next_parameter
180
180
  parameter_candidate = @tokens.first
181
181
  parameter = (parameter_candidate.nil? || OPTION_REGEXP =~ parameter_candidate || ALIASES_REGEXP =~ parameter_candidate) ? nil : @tokens.shift
182
182
  return parameter
183
183
  end
184
-
184
+
185
185
  def parse!(tokens)
186
- @current_definition = @definition
186
+ @current_definition = @definition
187
187
  @first_token = true
188
188
  @tokens = tokens.clone
189
-
189
+
190
190
  @options = {}
191
191
  @parameters = []
192
192
  @command = nil
193
-
193
+
194
194
  prepare_result!
195
-
195
+
196
196
  while next_token
197
-
197
+
198
198
  if @first_token && command_definition = @definition.has_command?(@current_token)
199
199
  @current_definition = command_definition
200
200
  @command = CommandLine::Option.rewrite(@current_token)
@@ -204,34 +204,34 @@ module CommandLine
204
204
  when OPTION_REGEXP; handle_option($1)
205
205
  else; handle_other_parameter(@current_token)
206
206
  end
207
- @first_token = false
207
+ @first_token = false
208
208
  end
209
-
209
+
210
210
  end
211
-
211
+
212
212
  validate_arguments!
213
-
214
- return self
213
+
214
+ return self
215
215
  end
216
-
216
+
217
217
  protected
218
-
218
+
219
219
  def prepare_result!
220
220
  multiple_options = Hash[*@current_definition.options.select { |name, o| o.multiple? }.flatten]
221
221
  multiple_options.each { |name, definition| @options[definition] = [] }
222
222
  end
223
-
223
+
224
224
  def validate_arguments!
225
225
  if @current_definition.parameters && !(@current_definition.parameters === @parameters.length)
226
226
  raise CommandLine::ParametersOutOfRange.new(@current_definition.parameters, @parameters.length)
227
227
  end
228
-
228
+
229
229
  required_options = Hash[*@current_definition.options.select { |name, o| o.required? }.flatten]
230
230
  required_options.each do |name, definition|
231
- raise CommandLine::RequiredOptionMissing, definition unless self[name]
231
+ raise CommandLine::RequiredOptionMissing, definition unless self[name]
232
232
  end
233
233
  end
234
-
234
+
235
235
  def handle_alias_expansion(aliases)
236
236
  aliases.reverse.scan(/./) do |alias_char|
237
237
  if option_definition = @current_definition[alias_char]
@@ -241,24 +241,24 @@ module CommandLine
241
241
  end
242
242
  end
243
243
  end
244
-
244
+
245
245
  def handle_other_parameter(parameter)
246
246
  @parameters << parameter
247
247
  end
248
-
248
+
249
249
  def handle_option(option_name)
250
250
  option_definition = @current_definition[option_name]
251
251
  raise CommandLine::UnknownOption, option_name if option_definition.nil?
252
-
252
+
253
253
  if option_definition.multiple?
254
- @options[option_definition] << option_definition.parse(self)
254
+ @options[option_definition] << option_definition.parse(self)
255
255
  else
256
256
  @options[option_definition] = option_definition.parse(self)
257
257
  end
258
258
  end
259
-
259
+
260
260
  end
261
-
261
+
262
262
  # Commandline parsing errors and exceptions
263
263
  class Error < Exception
264
264
  end
@@ -267,7 +267,7 @@ module CommandLine
267
267
  class RequiredOptionMissing < CommandLine::Error
268
268
  def initialize(option)
269
269
  super("You have to provide the #{option.name} option!")
270
- end
270
+ end
271
271
  end
272
272
 
273
273
  # Missing a required file
@@ -282,14 +282,14 @@ module CommandLine
282
282
  else
283
283
  super("The command expected #{expected} parameters, but found #{actual}!")
284
284
  end
285
- end
285
+ end
286
286
  end
287
287
 
288
288
  # Missing a required flag argument
289
289
  class ParameterExpected < CommandLine::Error
290
290
  def initialize(option)
291
291
  super("The option #{option.inspect} expects a parameter!")
292
- end
292
+ end
293
293
  end
294
294
 
295
295
  # Encountered an unkown flag
@@ -297,5 +297,5 @@ module CommandLine
297
297
  def initialize(option_identifier)
298
298
  super("#{option_identifier.inspect} not recognized as a valid option!")
299
299
  end
300
- end
300
+ end
301
301
  end
@@ -1,12 +1,12 @@
1
1
 
2
2
  class DatabaseConsole
3
-
3
+
4
4
  IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
5
-
5
+
6
6
  def initialize(arguments)
7
7
  @arguments = arguments
8
8
  end
9
-
9
+
10
10
  def run!
11
11
  libraries = ['irb/completion', 'rubygems', './lib/request_log_analyzer', './lib/cli/database_console_init']
12
12
  libaries_string = libraries.map { |l| "-r #{l}" }.join(' ')
@@ -23,7 +23,7 @@ class Request
23
23
  def inspect
24
24
  request_inspect = "Request[id: #{id}]"
25
25
  request_inspect << " <#{lines.first.source.filename}>" if lines.first.source
26
-
26
+
27
27
  inspected_lines = lines.map do |line|
28
28
  inspect_line = " - #{line.line_type} (line #{line.lineno})"
29
29
  if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
@@ -32,7 +32,7 @@ class Request
32
32
  end
33
33
  inspect_line
34
34
  end
35
-
35
+
36
36
  request_inspect << "\n" << inspected_lines.join("\n") << "\n\n"
37
37
  end
38
38
  end
@@ -37,8 +37,8 @@ module CommandLine
37
37
  private
38
38
  def fmt_bar
39
39
  bar_width = do_percentage * @terminal_width / 100
40
- sprintf("[%s%s]",
41
- @bar_mark * bar_width,
40
+ sprintf("[%s%s]",
41
+ @bar_mark * bar_width,
42
42
  " " * (@terminal_width - bar_width))
43
43
  end
44
44
 
@@ -51,9 +51,9 @@ module CommandLine
51
51
  end
52
52
 
53
53
  def fmt_stat_for_file_transfer
54
- if @finished_p then
54
+ if @finished_p then
55
55
  sprintf("%s %s %s", bytes, transfer_rate, elapsed)
56
- else
56
+ else
57
57
  sprintf("%s %s %s", bytes, transfer_rate, eta)
58
58
  end
59
59
  end
@@ -106,7 +106,7 @@ module CommandLine
106
106
  elapsed = Time.now - @start_time
107
107
  sprintf("Time: %s", format_time(elapsed))
108
108
  end
109
-
109
+
110
110
  def eol
111
111
  if @finished_p then "\n" else "\r" end
112
112
  end
@@ -120,14 +120,14 @@ module CommandLine
120
120
  end
121
121
 
122
122
  def show
123
- arguments = @format_arguments.map {|method|
123
+ arguments = @format_arguments.map {|method|
124
124
  method = sprintf("fmt_%s", method)
125
125
  send(method)
126
126
  }
127
127
  line = sprintf(@format, *arguments)
128
128
 
129
129
  width = terminal_width(80)
130
- if line.length == width - 1
130
+ if line.length == width - 1
131
131
  @out.print(line + eol)
132
132
  @out.flush
133
133
  elsif line.length >= width
@@ -150,7 +150,7 @@ module CommandLine
150
150
  end
151
151
 
152
152
  # Use "!=" instead of ">" to support negative changes
153
- if cur_percentage != prev_percentage ||
153
+ if cur_percentage != prev_percentage ||
154
154
  Time.now - @previous_time >= 1 || @finished_p
155
155
  show
156
156
  end
@@ -198,9 +198,9 @@ module CommandLine
198
198
  end
199
199
 
200
200
  def set (count)
201
- count = 0 if count < 0
201
+ count = 0 if count < 0
202
202
  count = @total if count > @total
203
-
203
+
204
204
  @current = count
205
205
  show_if_needed
206
206
  @previous = @current