request-log-analyzer 1.9.9 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore CHANGED
@@ -8,4 +8,6 @@ requests.db
8
8
  /tmp
9
9
  /classes
10
10
  /files
11
- /coverage
11
+ /coverage
12
+ Gemfile.lock
13
+ /.bundle/
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source :rubygems
2
+ gemspec
@@ -20,11 +20,11 @@ module RequestLogAnalyzer
20
20
  # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
21
21
  # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
22
22
  def self.build_from_arguments(arguments)
23
-
23
+
24
24
  require 'mixins/gets_memory_protection' if arguments[:gets_memory_protection]
25
-
25
+
26
26
  options = {}
27
-
27
+
28
28
  # Copy fields
29
29
  options[:database] = arguments[:database]
30
30
  options[:reset_database] = arguments[:reset_database]
@@ -33,7 +33,7 @@ module RequestLogAnalyzer
33
33
  options[:mail] = arguments[:mail]
34
34
  options[:no_progress] = arguments[:no_progress]
35
35
  options[:format] = arguments[:format]
36
- options[:output] = arguments[:output].downcase
36
+ options[:output] = arguments[:output]
37
37
  options[:file] = arguments[:file]
38
38
  options[:after] = arguments[:after]
39
39
  options[:before] = arguments[:before]
@@ -44,25 +44,25 @@ module RequestLogAnalyzer
44
44
  options[:report_width] = arguments[:report_width]
45
45
  options[:report_sort] = arguments[:report_sort]
46
46
  options[:report_amount] = arguments[:report_amount]
47
- options[:mailhost] = arguments[:mailhost]
47
+ options[:mailhost] = arguments[:mailhost]
48
48
  options[:mailsubject] = arguments[:mailsubject]
49
- options[:silent] = arguments[:silent]
50
- options[:parse_strategy] = arguments[:parse_strategy]
51
-
49
+ options[:silent] = arguments[:silent]
50
+ options[:parse_strategy] = arguments[:parse_strategy]
51
+
52
52
  # Apache format workaround
53
53
  if arguments[:rails_format]
54
54
  options[:format] = {:rails => arguments[:rails_format]}
55
55
  elsif arguments[:apache_format]
56
56
  options[:format] = {:apache => arguments[:apache_format]}
57
57
  end
58
-
58
+
59
59
  # Handle output format casing
60
60
  if options[:output].class == String
61
61
  options[:output] = 'FancyHTML' if options[:output] =~ /^fancy_?html$/i
62
62
  options[:output] = 'HTML' if options[:output] =~ /^html$/i
63
63
  options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
64
64
  end
65
-
65
+
66
66
  # Register sources
67
67
  if arguments.parameters.length == 1
68
68
  file = arguments.parameters[0]
@@ -77,7 +77,7 @@ module RequestLogAnalyzer
77
77
  else
78
78
  options.store(:source_files, arguments.parameters)
79
79
  end
80
-
80
+
81
81
  # Guess file format
82
82
  if !options[:format] && options[:source_files]
83
83
  options[:format] = :rails # Default
@@ -91,7 +91,7 @@ module RequestLogAnalyzer
91
91
  end
92
92
  end
93
93
  end
94
-
94
+
95
95
  build(options)
96
96
  end
97
97
 
@@ -124,7 +124,7 @@ module RequestLogAnalyzer
124
124
  # RequestLogAnalyzer::Controller.build(
125
125
  # :output => :HTML,
126
126
  # :mail => 'root@localhost',
127
- # :after => Time.now - 24*60*60,
127
+ # :after => Time.now - 24*60*60,
128
128
  # :source_files => '/var/log/passenger.log'
129
129
  # ).run!
130
130
  #
@@ -141,15 +141,15 @@ module RequestLogAnalyzer
141
141
  options[:report_sort] ||= 'sum,mean'
142
142
  options[:boring] ||= false
143
143
  options[:silent] ||= false
144
-
144
+
145
145
  options[:no_progress] = true if options[:silent]
146
-
146
+
147
147
  # Deprecation warnings
148
148
  if options[:dump]
149
149
  warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead."
150
150
  options[:yaml] = options[:dump]
151
151
  end
152
-
152
+
153
153
  # Set the output class
154
154
  output_args = {}
155
155
  output_object = nil
@@ -158,10 +158,10 @@ module RequestLogAnalyzer
158
158
  else
159
159
  output_class = RequestLogAnalyzer::Output.const_get(options[:output])
160
160
  end
161
-
161
+
162
162
  output_sort = options[:report_sort].split(',').map { |s| s.to_sym }
163
163
  output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
164
-
164
+
165
165
  if options[:file]
166
166
  output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
167
167
  output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
@@ -173,26 +173,26 @@ module RequestLogAnalyzer
173
173
  output_args = {:width => options[:report_width].to_i, :color => !options[:boring],
174
174
  :characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
175
175
  end
176
-
176
+
177
177
  output_instance = output_class.new(output_object, output_args)
178
-
178
+
179
179
  # Create the controller with the correct file format
180
180
  if options[:format].kind_of?(Hash)
181
181
  file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
182
182
  else
183
183
  file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
184
184
  end
185
-
185
+
186
186
  # Kickstart the controller
187
- controller =
188
- Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format,
189
- :source_files => options[:source_files],
187
+ controller =
188
+ Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format,
189
+ :source_files => options[:source_files],
190
190
  :parse_strategy => options[:parse_strategy]),
191
191
  { :output => output_instance,
192
192
  :database => options[:database], # FUGLY!
193
- :yaml => options[:yaml],
193
+ :yaml => options[:yaml],
194
194
  :reset_database => options[:reset_database],
195
- :no_progress => options[:no_progress],
195
+ :no_progress => options[:no_progress],
196
196
  :silent => options[:silent]
197
197
  })
198
198
 
@@ -230,7 +230,7 @@ module RequestLogAnalyzer
230
230
 
231
231
  file_format.setup_environment(controller)
232
232
  return controller
233
- end
233
+ end
234
234
 
235
235
  # Builds a new Controller for the given log file format.
236
236
  # <tt>format</tt> Logfile format. Defaults to :rails
@@ -248,7 +248,7 @@ module RequestLogAnalyzer
248
248
  @filters = []
249
249
  @output = options[:output]
250
250
  @interrupted = false
251
-
251
+
252
252
  # Register the request format for this session after checking its validity
253
253
  raise "Invalid file format!" unless @source.file_format.valid?
254
254
 
@@ -30,20 +30,23 @@ module RequestLogAnalyzer::FileFormat
30
30
 
31
31
  # A hash that defines how the log format directives should be parsed.
32
32
  LOG_DIRECTIVES = {
33
- '%' => { :regexp => '%', :captures => [] },
34
- 'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)', :captures => [{:name => :remote_host, :type => :string}] },
35
- 'a' => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] },
36
- 'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
37
- 'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
38
- 'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
39
- 'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
40
- 'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
41
- 't' => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] },
42
- 's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
43
- 'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
44
- 'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
45
- 'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
46
- {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
33
+ '%' => { nil => { :regexp => '%', :captures => [] } },
34
+ 'h' => { nil => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)', :captures => [{:name => :remote_host, :type => :string}] } },
35
+ 'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
36
+ 'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
37
+ 'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },
38
+ 'D' => { nil => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
39
+ 'micro' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
40
+ 'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
41
+ },
42
+ 'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
43
+ 'T' => { nil => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
44
+ 't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
45
+ 's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
46
+ 'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
47
+ 'U' => { nil => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] } },
48
+ 'r' => { nil => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
49
+ {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]} },
47
50
  'i' => { 'Referer' => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
48
51
  'User-agent' => { :regexp => '(.*)', :captures => [{:name => :user_agent, :type => :user_agent}] }
49
52
  }
@@ -72,14 +75,13 @@ module RequestLogAnalyzer::FileFormat
72
75
 
73
76
  if variable
74
77
  # Check if we recognize the log directive
75
- directive = LOG_DIRECTIVES[variable]
76
- directive = directive[arg] if directive && arg
78
+ directive = LOG_DIRECTIVES[variable][arg] rescue nil
77
79
 
78
80
  if directive
79
81
  line_regexp << directive[:regexp] # Parse the value of the directive
80
82
  captures += directive[:captures] # Add the directive's information to the captures
81
83
  else
82
- puts "%#{directive} log directiven not yet supported, field is ignored."
84
+ puts "Apache log directive %#{arg}#{variable} is not yet supported by RLA, the field will be ignored."
83
85
  line_regexp << '.*' # Just accept any input for this literal
84
86
  end
85
87
  end
@@ -126,22 +128,11 @@ module RequestLogAnalyzer::FileFormat
126
128
  "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
127
129
  end
128
130
 
129
- # This function can be overridden to rewrite the path for better categorization in the
130
- # reports.
131
- def convert_path(value, definition)
132
- value
133
- end
134
-
135
131
  # This function can be overridden to simplify the user agent string for better
136
132
  # categorization in the reports
137
133
  def convert_user_agent(value, definition)
138
134
  value # TODO
139
135
  end
140
-
141
- # Make sure that the string '-' is parsed as a nil value.
142
- def convert_nillable_string(value, definition)
143
- value == '-' ? nil : value
144
- end
145
136
  end
146
137
  end
147
138
  end
@@ -0,0 +1,138 @@
1
+ module RequestLogAnalyzer::FileFormat
2
+
3
+ class Haproxy < RequestLogAnalyzer::FileFormat::Base
4
+
5
+ extend CommonRegularExpressions
6
+
7
+ # Define line types
8
+ line_definition :haproxy do |line|
9
+ line.header = true
10
+ line.footer = true
11
+
12
+ line.regexp = %r{
13
+ (#{ip_address}):\d+\s # client_ip ':' client_port
14
+ \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})\.\d{3}\]\s # '[' accept_date ']'
15
+ (\S+)\s # frontend_name
16
+ (\S+)\/(\S+)\s # backend_name '/' server_name
17
+ (\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s # Tq '/' Tw '/' Tc '/' Tr '/' Tt
18
+ (\d+)\s # status_code
19
+ \+?(\d+)\s # bytes_read
20
+ (\S+)\s # captured_request_cookie
21
+ (\S+)\s # captured_response_cookie
22
+ (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
23
+ (\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
24
+ (\d+)\/(\d+)\s # srv_queue '/' backend_queue
25
+ (\S*)\s? # captured_request_headers
26
+ (\S*)\s? # captured_response_headers
27
+ "([^"]*)" # '"' http_request '"'
28
+ }x
29
+
30
+ line.capture(:client_ip).as(:string)
31
+ line.capture(:accept_date).as(:timestamp)
32
+ line.capture(:frontend_name).as(:string)
33
+ line.capture(:backend_name).as(:string)
34
+ line.capture(:server_name).as(:string)
35
+ line.capture(:tq).as(:nillable_duration, :unit => :msec)
36
+ line.capture(:tw).as(:nillable_duration, :unit => :msec)
37
+ line.capture(:tc).as(:nillable_duration, :unit => :msec)
38
+ line.capture(:tr).as(:nillable_duration, :unit => :msec)
39
+ line.capture(:tt).as(:duration, :unit => :msec)
40
+ line.capture(:status_code).as(:integer)
41
+ line.capture(:bytes_read).as(:traffic, :unit => :byte)
42
+ line.capture(:captured_request_cookie).as(:nillable_string)
43
+ line.capture(:captured_response_cookie).as(:nillable_string)
44
+ line.capture(:termination_event_code).as(:nillable_string)
45
+ line.capture(:terminated_session_state).as(:nillable_string)
46
+ line.capture(:clientside_persistence_cookie).as(:nillable_string)
47
+ line.capture(:serverside_persistence_cookie).as(:nillable_string)
48
+ line.capture(:actconn).as(:integer)
49
+ line.capture(:feconn).as(:integer)
50
+ line.capture(:beconn).as(:integer)
51
+ line.capture(:srv_conn).as(:integer)
52
+ line.capture(:retries).as(:integer)
53
+ line.capture(:srv_queue).as(:integer)
54
+ line.capture(:backend_queue).as(:integer)
55
+ line.capture(:captured_request_headers).as(:nillable_string)
56
+ line.capture(:captured_response_headers).as(:nillable_string)
57
+ line.capture(:http_request).as(:nillable_string)
58
+ end
59
+
60
+ # Define the summary report
61
+ report do |analyze|
62
+ analyze.hourly_spread :field => :accept_date
63
+
64
+ analyze.frequency :client_ip,
65
+ :title => "Hits per IP"
66
+
67
+ analyze.frequency :frontend_name,
68
+ :title => "Hits per frontend service"
69
+
70
+ analyze.frequency :backend_name,
71
+ :title => "Hits per backend service"
72
+
73
+ analyze.frequency :server_name,
74
+ :title => "Hits per backend server"
75
+
76
+ analyze.frequency :status_code,
77
+ :title => "HTTP response code frequency"
78
+
79
+ analyze.frequency :http_request,
80
+ :title => "Most popular requests"
81
+
82
+ analyze.frequency :http_request,
83
+ :title => "Most frequent HTTP 40x errors",
84
+ :category => lambda { |r| "#{r[:http_request]}"},
85
+ :if => lambda { |r| r[:status_code] >= 400 and r[:status_code] <= 417 }
86
+
87
+ analyze.frequency :http_request,
88
+ :title => "Most frequent HTTP 50x errors",
89
+ :category => lambda { |r| "#{r[:http_request]}"},
90
+ :if => lambda { |r| r[:status_code] >= 500 and r[:status_code] <= 505 }
91
+
92
+ analyze.traffic :bytes_read,
93
+ :title => "Traffic per frontend service",
94
+ :category => lambda { |r| "#{r[:frontend_name]}"}
95
+
96
+ analyze.traffic :bytes_read,
97
+ :title => "Traffic per backend service",
98
+ :category => lambda { |r| "#{r[:backend_name]}"}
99
+
100
+ analyze.traffic :bytes_read,
101
+ :title => "Traffic per backend server",
102
+ :category => lambda { |r| "#{r[:server_name]}"}
103
+
104
+ analyze.duration :tr,
105
+ :title => "Time waiting for backend response",
106
+ :category => lambda { |r| "#{r[:http_request]}"}
107
+
108
+ analyze.duration :tt,
109
+ :title => "Total time spent on request",
110
+ :category => lambda { |r| "#{r[:http_request]}"}
111
+ end
112
+
113
+ # Define a custom Request class for the HAProxy file format to speed up
114
+ # timestamp handling. Shamelessly copied from apache.rb
115
+ class Request < RequestLogAnalyzer::Request
116
+
117
+ MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
118
+ 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
119
+
120
+ # Do not use DateTime.parse, but parse the timestamp ourselves to return
121
+ # a integer to speed up parsing.
122
+ def convert_timestamp(value, definition)
123
+ "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
124
+ end
125
+
126
+ # Make sure that the strings '-' or '{}' or '' are parsed as a nil value.
127
+ def convert_nillable_string(value, definition)
128
+ value =~ /-|\{\}|^$/ ? nil : value
129
+ end
130
+
131
+ # Make sure that -1 is parsed as a nil value.
132
+ def convert_nillable_duration(value, definition)
133
+ value == '-1' ? nil : convert_duration(value, definition)
134
+ end
135
+
136
+ end
137
+ end
138
+ end
@@ -1,24 +1,25 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
2
 
3
+ # PostgresQL spec 8.3.7
3
4
  class Postgresql < Base
4
5
 
5
6
  extend CommonRegularExpressions
6
7
 
7
8
  line_definition :query do |line|
8
9
  line.header = true
9
- line.teaser = /LOG\: query\:/
10
- line.regexp = /(#{timestamp('%y-%m-%d %k:%M:%S')})\ LOG: query:\s+(.*)/
10
+ line.teaser = /.*LOG\:/
11
+ line.regexp = /(#{timestamp('%Y-%m-%d %k:%M:%S')})\ \S+ \[\d+\]\:\ \[.*\]\ LOG\:\ \ \d+\:\ duration\: (.*)\ ms\ \ statement:\ (.*)/
11
12
 
12
13
  line.capture(:timestamp).as(:timestamp)
14
+ line.capture(:query_time).as(:duration, :unit => :sec)
13
15
  line.capture(:query_fragment)
14
16
  end
15
17
 
16
- line_definition :duration do |line|
18
+ line_definition :location do |line|
17
19
  line.footer = true
18
- line.teaser = /duration:/
19
- line.regexp = /#{timestamp('%y-%m-%d %k:%M:%S')}\ LOG\: duration\: (.*)(\ )sec/
20
+ line.teaser = /.*LOCATION:/
21
+ line.regexp = /.*(\ )LOCATION:/
20
22
 
21
- line.capture(:query_time).as(:duration, :unit => :sec)
22
23
  line.capture(:query).as(:sql) # Hack to gather up fragments
23
24
  end
24
25
 
@@ -36,7 +36,7 @@ module RequestLogAnalyzer::FileFormat
36
36
  line_definition :completed do |line|
37
37
  line.footer = true
38
38
  line.teaser = /Completed /
39
- line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms\))?/
39
+ line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
40
40
 
41
41
  line.capture(:status).as(:integer)
42
42
  line.capture(:duration).as(:duration, :unit => :msec)
@@ -0,0 +1,56 @@
1
+ module RequestLogAnalyzer::FileFormat
2
+
3
+ # FileFormat for W3C access logs.
4
+ class W3c < Base
5
+
6
+ extend CommonRegularExpressions
7
+
8
+ line_definition :access do |line|
9
+ line.header = true
10
+ line.footer = true
11
+ line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) ([\w|\/|.]+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/
12
+
13
+ line.capture(:timestamp).as(:timestamp)
14
+ line.capture(:remote_ip)
15
+ line.capture(:username).as(:nillable_string)
16
+ line.capture(:local_ip)
17
+ line.capture(:port).as(:integer)
18
+ line.capture(:method)
19
+ line.capture(:path).as(:path)
20
+ line.capture(:http_status).as(:integer)
21
+ line.capture(:bytes_sent).as(:traffic, :unit => :byte)
22
+ line.capture(:bytes_received).as(:traffic, :unit => :byte)
23
+ line.capture(:duration).as(:duration, :unit => :msec)
24
+ line.capture(:user_agent)
25
+ line.capture(:referer)
26
+ end
27
+
28
+ report do |analyze|
29
+ analyze.timespan
30
+ analyze.hourly_spread
31
+
32
+ analyze.frequency :category => :http_method, :title => "HTTP methods"
33
+ analyze.frequency :category => :http_status, :title => "HTTP statuses"
34
+
35
+ analyze.frequency :category => :path, :title => "Most popular URIs"
36
+
37
+ analyze.frequency :category => :user_agent, :title => "User agents"
38
+ analyze.frequency :category => :referer, :title => "Referers"
39
+
40
+ analyze.duration :duration => :duration, :category => :path, :title => 'Request duration'
41
+ analyze.traffic :traffic => :bytes_sent, :category => :path, :title => 'Traffic out'
42
+ analyze.traffic :traffic => :bytes_received, :category => :path, :title => 'Traffic in'
43
+ end
44
+
45
+ class Request < RequestLogAnalyzer::Request
46
+ # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
47
+ # to speed up parsing.
48
+ def convert_timestamp(value, definition)
49
+ "#{value[0,4]}#{value[5,2]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
50
+ end
51
+ end
52
+
53
+
54
+ end
55
+
56
+ end
@@ -13,6 +13,8 @@ module RequestLogAnalyzer::FileFormat
13
13
  autoload :DelayedJob21, 'request_log_analyzer/file_format/delayed_job21'
14
14
  autoload :Apache, 'request_log_analyzer/file_format/apache'
15
15
  autoload :AmazonS3, 'request_log_analyzer/file_format/amazon_s3'
16
+ autoload :W3c, 'request_log_analyzer/file_format/w3c'
17
+ autoload :Haproxy, 'request_log_analyzer/file_format/haproxy'
16
18
 
17
19
  # Loads a FileFormat::Base subclass instance.
18
20
  # You can provide:
@@ -293,4 +295,4 @@ module RequestLogAnalyzer::FileFormat
293
295
  return nil
294
296
  end
295
297
  end
296
- end
298
+ end
@@ -27,6 +27,13 @@ module RequestLogAnalyzer
27
27
  def convert_integer(value, capture_definition); value.to_i; end
28
28
  def convert_sym(value, capture_definition); value.to_sym; end
29
29
  def convert_symbol(value, capture_definition); value.to_sym; end
30
+ def convert_nillable_string(value, definition); value == '-' ? nil : value ; end
31
+
32
+ # This function can be overridden to rewrite the path for better categorization in the
33
+ # reports.
34
+ def convert_path(value, definition)
35
+ value
36
+ end
30
37
 
31
38
  # Converts :eval field, which should evaluate to a hash.
32
39
  def convert_eval(value, capture_definition)
@@ -13,7 +13,7 @@ module RequestLogAnalyzer
13
13
 
14
14
  # The current version of request-log-analyzer.
15
15
  # Do not change the value by hand; it will be updated automatically by the gem release script.
16
- VERSION = "1.9.9"
16
+ VERSION = "1.10.0"
17
17
 
18
18
 
19
19
  autoload :Controller, 'request_log_analyzer/controller'
@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
2
2
  s.name = "request-log-analyzer"
3
3
 
4
4
  # Do not set the version and date field manually, this is done by the release script
5
- s.version = "1.9.9"
6
- s.date = "2010-12-17"
5
+ s.version = "1.10.0"
6
+ s.date = "2011-02-26"
7
7
 
8
8
  s.rubyforge_project = 'r-l-a'
9
9
 
@@ -39,6 +39,6 @@ Gem::Specification.new do |s|
39
39
 
40
40
  # The files and test_files directives are set automatically by the release script.
41
41
  # Do not change them by hand, but make sure to add the files to the git repository.
42
- s.files = %w(.gitignore DESIGN.rdoc LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
43
- s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
42
+ s.files = %w(.gitignore DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
43
+ s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
44
44
  end