request-log-analyzer 1.10.1 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +0 -1
- data/lib/request_log_analyzer.rb +15 -29
- data/lib/request_log_analyzer/aggregator.rb +5 -5
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +2 -1
- data/lib/request_log_analyzer/controller.rb +0 -3
- data/lib/request_log_analyzer/database.rb +6 -7
- data/lib/request_log_analyzer/file_format.rb +42 -13
- data/lib/request_log_analyzer/file_format/apache.rb +1 -1
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +2 -2
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +2 -2
- data/lib/request_log_analyzer/file_format/haproxy.rb +107 -13
- data/lib/request_log_analyzer/file_format/mysql.rb +5 -5
- data/lib/request_log_analyzer/file_format/rails3.rb +7 -0
- data/lib/request_log_analyzer/filter.rb +4 -5
- data/lib/request_log_analyzer/line_definition.rb +6 -4
- data/lib/request_log_analyzer/output.rb +3 -5
- data/lib/request_log_analyzer/source.rb +3 -4
- data/lib/request_log_analyzer/source/log_parser.rb +56 -4
- data/lib/request_log_analyzer/tracker.rb +8 -8
- data/request-log-analyzer.gemspec +3 -3
- data/spec/fixtures/mysql_slow_query.log +0 -1
- data/spec/integration/command_line_usage_spec.rb +0 -5
- data/spec/lib/helpers.rb +2 -2
- data/spec/lib/matchers.rb +38 -7
- data/spec/lib/mocks.rb +1 -5
- data/spec/unit/database/base_class_spec.rb +1 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +58 -55
- data/spec/unit/file_format/apache_format_spec.rb +74 -162
- data/spec/unit/file_format/common_regular_expressions_spec.rb +51 -26
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -31
- data/spec/unit/file_format/delayed_job2_format_spec.rb +27 -32
- data/spec/unit/file_format/delayed_job_format_spec.rb +44 -63
- data/spec/unit/file_format/haproxy_format_spec.rb +69 -71
- data/spec/unit/file_format/line_definition_spec.rb +26 -33
- data/spec/unit/file_format/merb_format_spec.rb +22 -37
- data/spec/unit/file_format/mysql_format_spec.rb +80 -123
- data/spec/unit/file_format/oink_format_spec.rb +29 -61
- data/spec/unit/file_format/postgresql_format_spec.rb +2 -4
- data/spec/unit/file_format/rack_format_spec.rb +49 -44
- data/spec/unit/file_format/rails3_format_spec.rb +17 -20
- data/spec/unit/file_format/rails_format_spec.rb +52 -68
- data/spec/unit/file_format/w3c_format_spec.rb +40 -39
- data/spec/unit/source/log_parser_spec.rb +1 -1
- metadata +4 -7
- data/lib/mixins/gets_memory_protection.rb +0 -80
- data/lib/request_log_analyzer/output/fancy_html.rb +0 -44
- data/lib/request_log_analyzer/source/database_loader.rb +0 -87
data/bin/request-log-analyzer
CHANGED
data/lib/request_log_analyzer.rb
CHANGED
@@ -10,36 +10,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
|
|
10
10
|
# The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
|
11
11
|
# is running.
|
12
12
|
module RequestLogAnalyzer
|
13
|
-
|
13
|
+
|
14
14
|
# The current version of request-log-analyzer.
|
15
15
|
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
16
|
-
VERSION = "1.
|
17
|
-
|
18
|
-
|
19
|
-
autoload :Controller, 'request_log_analyzer/controller'
|
20
|
-
autoload :Aggregator, 'request_log_analyzer/aggregator'
|
21
|
-
autoload :Database, 'request_log_analyzer/database'
|
22
|
-
autoload :FileFormat, 'request_log_analyzer/file_format'
|
23
|
-
autoload :Filter, 'request_log_analyzer/filter'
|
24
|
-
autoload :LineDefinition, 'request_log_analyzer/line_definition'
|
25
|
-
autoload :LogProcessor, 'request_log_analyzer/log_processor'
|
26
|
-
autoload :Mailer, 'request_log_analyzer/mailer'
|
27
|
-
autoload :Output, 'request_log_analyzer/output'
|
28
|
-
autoload :Request, 'request_log_analyzer/request'
|
29
|
-
autoload :Source, 'request_log_analyzer/source'
|
30
|
-
autoload :Tracker, 'request_log_analyzer/tracker'
|
31
|
-
|
32
|
-
# Loads constants that reside in the RequestLogAnalyzer tree using the constant name and its base
|
33
|
-
# constant to determine the filename.
|
34
|
-
# @param [Module] base The base constant to load the constant from. This should be <tt>Foo</tt> when
|
35
|
-
# the constant <tt>Foo::Bar</tt> is being loaded.
|
36
|
-
# @param [Symbol] const The constant to load from the base constant as a string or symbol. This
|
37
|
-
# should be <tt>"Bar"<tt> or <tt>:Bar</tt> when the constant <tt>Foo::Bar</tt> is being loaded.
|
38
|
-
# @return [Module] The loaded module, nil if it was not found on the expected location.
|
39
|
-
def self.load_default_class_file(base, const)
|
40
|
-
require "#{to_underscore("#{base.name}::#{const}")}"
|
41
|
-
base.const_get(const) if base.const_defined?(const)
|
42
|
-
end
|
16
|
+
VERSION = "1.11.0"
|
43
17
|
|
44
18
|
# Convert a string/symbol in camelcase ({RequestLogAnalyzer::Controller}) to underscores
|
45
19
|
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
@@ -55,9 +29,21 @@ module RequestLogAnalyzer
|
|
55
29
|
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
56
30
|
# filename.
|
57
31
|
#
|
58
|
-
# @param [#to_s] str The string-like to convert in the
|
32
|
+
# @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
|
59
33
|
# @return [String] The input string converted to camelcase form.
|
60
34
|
def self.to_camelcase(str)
|
61
35
|
str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
|
62
36
|
end
|
63
37
|
end
|
38
|
+
|
39
|
+
require 'request_log_analyzer/controller'
|
40
|
+
require 'request_log_analyzer/aggregator'
|
41
|
+
require 'request_log_analyzer/file_format'
|
42
|
+
require 'request_log_analyzer/filter'
|
43
|
+
require 'request_log_analyzer/line_definition'
|
44
|
+
require 'request_log_analyzer/log_processor'
|
45
|
+
require 'request_log_analyzer/mailer'
|
46
|
+
require 'request_log_analyzer/output'
|
47
|
+
require 'request_log_analyzer/request'
|
48
|
+
require 'request_log_analyzer/source'
|
49
|
+
require 'request_log_analyzer/tracker'
|
@@ -1,9 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer::Aggregator
|
2
2
|
|
3
|
-
autoload :Echo, 'request_log_analyzer/aggregator/echo'
|
4
|
-
autoload :Summarizer, 'request_log_analyzer/aggregator/summarizer'
|
5
|
-
autoload :DatabaseInserter, 'request_log_analyzer/aggregator/database_inserter'
|
6
|
-
|
7
3
|
# The base class of an aggregator. This class provides the interface to which
|
8
4
|
# every aggregator should comply (by simply subclassing this class).
|
9
5
|
class Base
|
@@ -46,4 +42,8 @@ module RequestLogAnalyzer::Aggregator
|
|
46
42
|
end
|
47
43
|
|
48
44
|
end
|
49
|
-
end
|
45
|
+
end
|
46
|
+
|
47
|
+
require 'request_log_analyzer/aggregator/echo'
|
48
|
+
require 'request_log_analyzer/aggregator/summarizer'
|
49
|
+
require 'request_log_analyzer/aggregator/database_inserter'
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
module RequestLogAnalyzer::Aggregator
|
3
2
|
|
4
3
|
# The database aggregator will create an SQLite3 database with all parsed request information.
|
@@ -19,6 +18,8 @@ module RequestLogAnalyzer::Aggregator
|
|
19
18
|
# Establishes a connection to the database and creates the necessary database schema for the
|
20
19
|
# current file format
|
21
20
|
def prepare
|
21
|
+
require 'request_log_analyzer/database'
|
22
|
+
|
22
23
|
@sources = {}
|
23
24
|
@database = RequestLogAnalyzer::Database.new(options[:database])
|
24
25
|
@database.file_format = source.file_format
|
@@ -21,8 +21,6 @@ module RequestLogAnalyzer
|
|
21
21
|
# <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
|
22
22
|
def self.build_from_arguments(arguments)
|
23
23
|
|
24
|
-
require 'mixins/gets_memory_protection' if arguments[:gets_memory_protection]
|
25
|
-
|
26
24
|
options = {}
|
27
25
|
|
28
26
|
# Copy fields
|
@@ -58,7 +56,6 @@ module RequestLogAnalyzer
|
|
58
56
|
|
59
57
|
# Handle output format casing
|
60
58
|
if options[:output].class == String
|
61
|
-
options[:output] = 'FancyHTML' if options[:output] =~ /^fancy_?html$/i
|
62
59
|
options[:output] = 'HTML' if options[:output] =~ /^html$/i
|
63
60
|
options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
|
64
61
|
end
|
@@ -3,12 +3,7 @@ require 'active_record'
|
|
3
3
|
|
4
4
|
class RequestLogAnalyzer::Database
|
5
5
|
|
6
|
-
|
7
|
-
autoload :Base, 'request_log_analyzer/database/base'
|
8
|
-
autoload :Request, 'request_log_analyzer/database/request'
|
9
|
-
autoload :Source, 'request_log_analyzer/database/source'
|
10
|
-
autoload :Warning, 'request_log_analyzer/database/warning'
|
11
|
-
|
6
|
+
require 'request_log_analyzer/database/connection'
|
12
7
|
include RequestLogAnalyzer::Database::Connection
|
13
8
|
|
14
9
|
attr_accessor :file_format
|
@@ -100,5 +95,9 @@ class RequestLogAnalyzer::Database
|
|
100
95
|
end
|
101
96
|
end
|
102
97
|
end
|
103
|
-
|
104
98
|
end
|
99
|
+
|
100
|
+
require 'request_log_analyzer/database/base'
|
101
|
+
require 'request_log_analyzer/database/request'
|
102
|
+
require 'request_log_analyzer/database/source'
|
103
|
+
require 'request_log_analyzer/database/warning'
|
@@ -121,6 +121,18 @@ module RequestLogAnalyzer::FileFormat
|
|
121
121
|
'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
|
122
122
|
'%' => '%'
|
123
123
|
}
|
124
|
+
|
125
|
+
# Creates a regular expression to match a hostname
|
126
|
+
def hostname(blank = false)
|
127
|
+
regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
|
128
|
+
add_blank_option(regexp, blank)
|
129
|
+
end
|
130
|
+
|
131
|
+
# Creates a regular expression to match a hostname or ip address
|
132
|
+
def hostname_or_ip_address(blank = false)
|
133
|
+
regexp = Regexp.union(hostname, ip_address)
|
134
|
+
add_blank_option(regexp, blank)
|
135
|
+
end
|
124
136
|
|
125
137
|
# Create a regular expression for a timestamp, generated by a strftime call.
|
126
138
|
# Provide the format string to construct a matching regular expression.
|
@@ -139,12 +151,7 @@ module RequestLogAnalyzer::FileFormat
|
|
139
151
|
end
|
140
152
|
end
|
141
153
|
|
142
|
-
|
143
|
-
return case blank
|
144
|
-
when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
|
145
|
-
when true then Regexp.union(regexp, //)
|
146
|
-
else regexp
|
147
|
-
end
|
154
|
+
add_blank_option(Regexp.new(regexp), blank)
|
148
155
|
end
|
149
156
|
|
150
157
|
# Construct a regular expression to parse IPv4 and IPv6 addresses.
|
@@ -162,12 +169,22 @@ module RequestLogAnalyzer::FileFormat
|
|
162
169
|
ipv6_regex_compressed_hex_4_dec = /(?:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::(?:(?:[0-9A-Fa-f]{1,4}:)*)#{ipv4_regexp}/
|
163
170
|
ipv6_regexp = Regexp.union(ipv6_regex_8_hex, ipv6_regex_compressed_hex, ipv6_regex_6_hex_4_dec, ipv6_regex_compressed_hex_4_dec)
|
164
171
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
172
|
+
add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
|
173
|
+
end
|
174
|
+
|
175
|
+
def anchored(regexp)
|
176
|
+
/^#{regexp}$/
|
177
|
+
end
|
178
|
+
|
179
|
+
protected
|
180
|
+
|
181
|
+
# Allow the field to be blank if this option is given. This can be true to
|
182
|
+
# allow an empty string or a string alternative for the nil value.
|
183
|
+
def add_blank_option(regexp, blank)
|
184
|
+
case blank
|
185
|
+
when String; Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
|
186
|
+
when true; Regexp.union(regexp, //)
|
187
|
+
else regexp
|
171
188
|
end
|
172
189
|
end
|
173
190
|
end
|
@@ -260,9 +277,12 @@ module RequestLogAnalyzer::FileFormat
|
|
260
277
|
end
|
261
278
|
|
262
279
|
# Checks whether the file format is valid so it can be safely used with RLA.
|
263
|
-
def
|
280
|
+
def well_formed?
|
264
281
|
valid_line_definitions? && valid_request_class?
|
265
282
|
end
|
283
|
+
|
284
|
+
alias_method :valid?, :well_formed?
|
285
|
+
|
266
286
|
|
267
287
|
# Checks whether the line definitions form a valid language.
|
268
288
|
# A file format should have at least a header and a footer line type
|
@@ -294,5 +314,14 @@ module RequestLogAnalyzer::FileFormat
|
|
294
314
|
|
295
315
|
return nil
|
296
316
|
end
|
317
|
+
|
318
|
+
# Returns the max line length for this file format if any.
|
319
|
+
def max_line_length
|
320
|
+
self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
|
321
|
+
end
|
322
|
+
|
323
|
+
def line_divider
|
324
|
+
self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
|
325
|
+
end
|
297
326
|
end
|
298
327
|
end
|
@@ -31,7 +31,7 @@ module RequestLogAnalyzer::FileFormat
|
|
31
31
|
# A hash that defines how the log format directives should be parsed.
|
32
32
|
LOG_DIRECTIVES = {
|
33
33
|
'%' => { nil => { :regexp => '%', :captures => [] } },
|
34
|
-
'h' => { nil => { :regexp =>
|
34
|
+
'h' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :remote_host, :type => :string}] } },
|
35
35
|
'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
|
36
36
|
'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
|
37
37
|
'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },
|
@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
|
|
8
8
|
|
9
9
|
line_definition :job_lock do |line|
|
10
10
|
line.header = true
|
11
|
-
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(
|
11
|
+
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
12
12
|
|
13
13
|
line.capture(:timestamp).as(:timestamp)
|
14
14
|
line.capture(:host)
|
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
|
|
18
18
|
|
19
19
|
line_definition :job_completed do |line|
|
20
20
|
line.footer = true
|
21
|
-
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(
|
21
|
+
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
|
22
22
|
line.capture(:timestamp).as(:timestamp)
|
23
23
|
line.capture(:host)
|
24
24
|
line.capture(:pid).as(:integer)
|
@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
|
|
8
8
|
|
9
9
|
line_definition :job_lock do |line|
|
10
10
|
line.header = true
|
11
|
-
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(
|
11
|
+
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
12
12
|
|
13
13
|
line.capture(:timestamp).as(:timestamp)
|
14
14
|
line.capture(:host)
|
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
|
|
18
18
|
|
19
19
|
line_definition :job_completed do |line|
|
20
20
|
line.footer = true
|
21
|
-
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(
|
21
|
+
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
|
22
22
|
line.capture(:timestamp).as(:timestamp)
|
23
23
|
line.capture(:host)
|
24
24
|
line.capture(:pid).as(:integer)
|
@@ -4,31 +4,45 @@ module RequestLogAnalyzer::FileFormat
|
|
4
4
|
|
5
5
|
extend CommonRegularExpressions
|
6
6
|
|
7
|
-
#
|
8
|
-
|
9
|
-
|
10
|
-
line.footer = true
|
11
|
-
|
12
|
-
line.regexp = %r{
|
7
|
+
# substitute version specific parts of the haproxy entry regexp.
|
8
|
+
def self.compose_regexp(millisecs, backends, counters, connections, queues)
|
9
|
+
%r{
|
13
10
|
(#{ip_address}):\d+\s # client_ip ':' client_port
|
14
|
-
\[(#{timestamp('%d/%b/%Y:%H:%M:%S')})
|
11
|
+
\[(#{timestamp('%d/%b/%Y:%H:%M:%S')})#{millisecs}\]\s # '[' accept_date ']'
|
15
12
|
(\S+)\s # frontend_name
|
16
|
-
|
17
|
-
|
13
|
+
#{backends}
|
14
|
+
#{counters}
|
18
15
|
(\d+)\s # status_code
|
19
16
|
\+?(\d+)\s # bytes_read
|
20
17
|
(\S+)\s # captured_request_cookie
|
21
18
|
(\S+)\s # captured_response_cookie
|
22
19
|
(\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
|
23
|
-
|
24
|
-
|
20
|
+
#{connections}
|
21
|
+
#{queues}
|
25
22
|
(\S*)\s? # captured_request_headers
|
26
23
|
(\S*)\s? # captured_response_headers
|
27
24
|
"([^"]*)" # '"' http_request '"'
|
28
25
|
}x
|
26
|
+
end
|
27
|
+
|
28
|
+
# Define line types
|
29
|
+
|
30
|
+
# line definition for haproxy 1.3 and higher
|
31
|
+
line_definition :haproxy13 do |line|
|
32
|
+
line.header = true
|
33
|
+
line.footer = true
|
34
|
+
line.teaser = /\.\d{3}\] \S+ \S+\/\S+ / # .millisecs] frontend_name backend_name/server_name
|
35
|
+
|
36
|
+
line.regexp = compose_regexp(
|
37
|
+
'\.\d{3}', # millisecs
|
38
|
+
'(\S+)\/(\S+)\s', # backend_name '/' server_name
|
39
|
+
'(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
|
40
|
+
'(\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s', # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
|
41
|
+
'(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
|
42
|
+
)
|
29
43
|
|
30
44
|
line.capture(:client_ip).as(:string)
|
31
|
-
line.capture(:
|
45
|
+
line.capture(:timestamp).as(:timestamp)
|
32
46
|
line.capture(:frontend_name).as(:string)
|
33
47
|
line.capture(:backend_name).as(:string)
|
34
48
|
line.capture(:server_name).as(:string)
|
@@ -57,9 +71,89 @@ module RequestLogAnalyzer::FileFormat
|
|
57
71
|
line.capture(:http_request).as(:nillable_string)
|
58
72
|
end
|
59
73
|
|
74
|
+
# haproxy 1.2 has a few fields less than 1.3+
|
75
|
+
line_definition :haproxy12 do |line|
|
76
|
+
line.header = true
|
77
|
+
line.footer = true
|
78
|
+
line.teaser = /\.\d{3}\] \S+ \S+ / # .millisecs] frontend_name server_name
|
79
|
+
|
80
|
+
line.regexp = compose_regexp(
|
81
|
+
'\.\d{3}', # millisecs
|
82
|
+
'(\S+)\s', # server_name
|
83
|
+
'(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
|
84
|
+
'(\d+)\/(\d+)\/(\d+)\s', # srv_conn '/' listener_conn '/' process_conn
|
85
|
+
'(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
|
86
|
+
)
|
87
|
+
|
88
|
+
line.capture(:client_ip).as(:string)
|
89
|
+
line.capture(:timestamp).as(:timestamp)
|
90
|
+
line.capture(:frontend_name).as(:string)
|
91
|
+
line.capture(:server_name).as(:string)
|
92
|
+
line.capture(:tq).as(:nillable_duration, :unit => :msec)
|
93
|
+
line.capture(:tw).as(:nillable_duration, :unit => :msec)
|
94
|
+
line.capture(:tc).as(:nillable_duration, :unit => :msec)
|
95
|
+
line.capture(:tr).as(:nillable_duration, :unit => :msec)
|
96
|
+
line.capture(:tt).as(:duration, :unit => :msec)
|
97
|
+
line.capture(:status_code).as(:integer)
|
98
|
+
line.capture(:bytes_read).as(:traffic, :unit => :byte)
|
99
|
+
line.capture(:captured_request_cookie).as(:nillable_string)
|
100
|
+
line.capture(:captured_response_cookie).as(:nillable_string)
|
101
|
+
line.capture(:termination_event_code).as(:nillable_string)
|
102
|
+
line.capture(:terminated_session_state).as(:nillable_string)
|
103
|
+
line.capture(:clientside_persistence_cookie).as(:nillable_string)
|
104
|
+
line.capture(:serverside_persistence_cookie).as(:nillable_string)
|
105
|
+
line.capture(:srv_conn).as(:integer)
|
106
|
+
line.capture(:listener_conn).as(:integer)
|
107
|
+
line.capture(:process_conn).as(:integer)
|
108
|
+
line.capture(:srv_queue).as(:integer)
|
109
|
+
line.capture(:backend_queue).as(:integer)
|
110
|
+
line.capture(:captured_request_headers).as(:nillable_string)
|
111
|
+
line.capture(:captured_response_headers).as(:nillable_string)
|
112
|
+
line.capture(:http_request).as(:nillable_string)
|
113
|
+
end
|
114
|
+
|
115
|
+
# and haproxy 1.1 has even less fields
|
116
|
+
line_definition :haproxy11 do |line|
|
117
|
+
line.header = true
|
118
|
+
line.footer = true
|
119
|
+
line.teaser = /:\d{2}\] \S+ \S+ / # :secs] frontend_name server_name
|
120
|
+
|
121
|
+
line.regexp = compose_regexp(
|
122
|
+
'', # no millisec precision in this version of haproxy
|
123
|
+
'(\S+)\s', # server_name
|
124
|
+
'(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tc '/' Tr '/' Tt
|
125
|
+
'(\d+)\/(\d+)\s', # listener_conn '/' process_conn
|
126
|
+
'' # no queues in this version of haproxy
|
127
|
+
)
|
128
|
+
|
129
|
+
line.capture(:client_ip).as(:string)
|
130
|
+
line.capture(:timestamp).as(:timestamp)
|
131
|
+
line.capture(:frontend_name).as(:string)
|
132
|
+
line.capture(:server_name).as(:string)
|
133
|
+
line.capture(:tq).as(:nillable_duration, :unit => :msec)
|
134
|
+
line.capture(:tc).as(:nillable_duration, :unit => :msec)
|
135
|
+
line.capture(:tr).as(:nillable_duration, :unit => :msec)
|
136
|
+
line.capture(:tt).as(:duration, :unit => :msec)
|
137
|
+
line.capture(:status_code).as(:integer)
|
138
|
+
line.capture(:bytes_read).as(:traffic, :unit => :byte)
|
139
|
+
line.capture(:captured_request_cookie).as(:nillable_string)
|
140
|
+
line.capture(:captured_response_cookie).as(:nillable_string)
|
141
|
+
line.capture(:termination_event_code).as(:nillable_string)
|
142
|
+
line.capture(:terminated_session_state).as(:nillable_string)
|
143
|
+
line.capture(:clientside_persistence_cookie).as(:nillable_string)
|
144
|
+
line.capture(:serverside_persistence_cookie).as(:nillable_string)
|
145
|
+
line.capture(:listener_conn).as(:integer)
|
146
|
+
line.capture(:process_conn).as(:integer)
|
147
|
+
line.capture(:srv_queue).as(:integer)
|
148
|
+
line.capture(:backend_queue).as(:integer)
|
149
|
+
line.capture(:captured_request_headers).as(:nillable_string)
|
150
|
+
line.capture(:captured_response_headers).as(:nillable_string)
|
151
|
+
line.capture(:http_request).as(:nillable_string)
|
152
|
+
end
|
153
|
+
|
60
154
|
# Define the summary report
|
61
155
|
report do |analyze|
|
62
|
-
analyze.hourly_spread :field => :
|
156
|
+
analyze.hourly_spread :field => :timestamp
|
63
157
|
|
64
158
|
analyze.frequency :client_ip,
|
65
159
|
:title => "Hits per IP"
|
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::FileFormat
|
|
15
15
|
line_definition :user_host do |line|
|
16
16
|
line.header = :alternative
|
17
17
|
line.teaser = /\# User\@Host\: /
|
18
|
-
line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (
|
18
|
+
line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (#{hostname(true)}) \[(#{ip_address(true)})\]/
|
19
19
|
|
20
20
|
line.capture(:user)
|
21
21
|
line.capture(:host)
|
@@ -34,18 +34,18 @@ module RequestLogAnalyzer::FileFormat
|
|
34
34
|
end
|
35
35
|
|
36
36
|
line_definition :use_database do |line|
|
37
|
-
line.regexp =
|
37
|
+
line.regexp = /^\s*use (\w+);\s*$/
|
38
38
|
line.capture(:database)
|
39
39
|
end
|
40
40
|
|
41
41
|
line_definition :query_part do |line|
|
42
|
-
line.regexp =
|
42
|
+
line.regexp = /^\s*(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
|
43
43
|
line.capture(:query_fragment)
|
44
44
|
end
|
45
45
|
|
46
46
|
line_definition :query do |line|
|
47
47
|
line.footer = true
|
48
|
-
line.regexp = /^(
|
48
|
+
line.regexp = /^(?!\s*(?:use |\# |SET ))(.*);\s*$/
|
49
49
|
line.capture(:query).as(:sql)
|
50
50
|
end
|
51
51
|
|
@@ -84,7 +84,7 @@ module RequestLogAnalyzer::FileFormat
|
|
84
84
|
sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list
|
85
85
|
sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list
|
86
86
|
|
87
|
-
return sql.
|
87
|
+
return sql.strip
|
88
88
|
end
|
89
89
|
|
90
90
|
def host
|