request-log-analyzer 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +10 -7
- data/lib/request_log_analyzer.rb +1 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +12 -1
- data/lib/request_log_analyzer/controller.rb +31 -10
- data/lib/request_log_analyzer/file_format.rb +46 -2
- data/lib/request_log_analyzer/file_format/apache.rb +1 -1
- data/lib/request_log_analyzer/file_format/mysql.rb +93 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +1 -1
- data/lib/request_log_analyzer/mailer.rb +7 -4
- data/lib/request_log_analyzer/output/fixed_width.rb +1 -1
- data/lib/request_log_analyzer/request.rb +5 -0
- data/lib/request_log_analyzer/source/log_parser.rb +33 -10
- data/lib/request_log_analyzer/tracker/count.rb +93 -0
- data/request-log-analyzer.gemspec +6 -6
- data/spec/fixtures/mysql_slow_query.log +110 -0
- data/spec/integration/mailer_spec.rb +179 -0
- data/spec/integration/scout_spec.rb +2 -1
- data/spec/lib/helpers.rb +21 -1
- data/spec/unit/file_format/format_autodetection_spec.rb +35 -0
- data/spec/unit/file_format/mysql_format_spec.rb +155 -0
- data/spec/unit/mailer_spec.rb +12 -0
- data/spec/unit/tracker/count_tracker_spec.rb +123 -0
- metadata +126 -115
data/bin/request-log-analyzer
CHANGED
@@ -27,14 +27,16 @@ begin
|
|
27
27
|
strip.switch(:keep_junk_lines, :j)
|
28
28
|
end
|
29
29
|
|
30
|
-
command_line.option(:format, :alias => :f
|
30
|
+
command_line.option(:format, :alias => :f)
|
31
31
|
command_line.option(:apache_format)
|
32
32
|
command_line.option(:rails_format)
|
33
33
|
|
34
34
|
command_line.option(:file, :alias => :e)
|
35
35
|
command_line.option(:mail, :alias => :m)
|
36
|
+
command_line.option(:mailhost, :default => 'localhost')
|
36
37
|
command_line.option(:parse_strategy, :default => 'assume-correct')
|
37
|
-
command_line.option(:
|
38
|
+
command_line.option(:yaml)
|
39
|
+
command_line.option(:dump) # To be deprecated
|
38
40
|
|
39
41
|
command_line.option(:aggregator, :alias => :a, :multiple => true)
|
40
42
|
|
@@ -48,7 +50,7 @@ begin
|
|
48
50
|
command_line.option(:before)
|
49
51
|
|
50
52
|
command_line.switch(:boring, :b)
|
51
|
-
command_line.option(:output, :alias => :o, :default => '
|
53
|
+
command_line.option(:output, :alias => :o, :default => 'fixedwidth')
|
52
54
|
command_line.option(:report_width, :default => terminal_width - 1)
|
53
55
|
command_line.option(:report_amount, :default => 20)
|
54
56
|
command_line.option(:report_sort, :default => 'sum,mean')
|
@@ -68,7 +70,7 @@ rescue CommandLine::Error => e
|
|
68
70
|
puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
|
69
71
|
puts
|
70
72
|
puts "Input options:"
|
71
|
-
puts " --format <format>, -f:
|
73
|
+
puts " --format <format>, -f: Log file format. amazon_s3, apache, merb, mysql, rack or rails (default)."
|
72
74
|
puts " --after <date> Only consider requests from <date> or later."
|
73
75
|
puts " --before <date> Only consider requests before <date>."
|
74
76
|
puts " --select <field> <value> Only consider requests where <field> matches <value>."
|
@@ -78,10 +80,11 @@ rescue CommandLine::Error => e
|
|
78
80
|
puts " --boring, -b Output reports without ASCII colors."
|
79
81
|
puts " --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
|
80
82
|
puts " --debug Print debug information while parsing."
|
81
|
-
puts " --file <filename>
|
83
|
+
puts " --file <filename> Redirect output to file."
|
82
84
|
puts " --mail <emailaddress> Send report to an email address."
|
83
|
-
puts " --
|
84
|
-
puts " --
|
85
|
+
puts " --mailhost <server> Use the given server as the SMTP server for sending email."
|
86
|
+
puts " --output <format> Output format. Supports 'html' and 'fixed_width' (default)."
|
87
|
+
puts " --yaml <filename> Dump the results in YAML format in the given file."
|
85
88
|
puts
|
86
89
|
puts "Examples:"
|
87
90
|
puts " request-log-analyzer development.log"
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -11,7 +11,7 @@ module RequestLogAnalyzer
|
|
11
11
|
|
12
12
|
# The current version of request-log-analyzer.
|
13
13
|
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
14
|
-
VERSION = "1.
|
14
|
+
VERSION = "1.5.0"
|
15
15
|
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
@@ -49,6 +49,17 @@ module RequestLogAnalyzer::Aggregator
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
+
# Track the frequency of a specific category
|
53
|
+
# <tt>category_field</tt> Field to track
|
54
|
+
# <tt>options</tt> options are passed to new frequency tracker
|
55
|
+
def count(category_field, options = {})
|
56
|
+
if category_field.kind_of?(Symbol)
|
57
|
+
track(:count, options.merge(:category => category_field))
|
58
|
+
elsif category_field.kind_of?(Hash)
|
59
|
+
track(:count, category_field.merge(options))
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
52
63
|
# Helper function to initialize a tracker and add it to the tracker array.
|
53
64
|
# <tt>tracker_class</tt> The class to include
|
54
65
|
# <tt>optiont</tt> The options to pass to the trackers.
|
@@ -145,7 +156,7 @@ module RequestLogAnalyzer::Aggregator
|
|
145
156
|
if has_log_ordering_warnings?
|
146
157
|
output.title("Parse warnings")
|
147
158
|
|
148
|
-
output.puts "Parseable lines were
|
159
|
+
output.puts "Parseable lines were encountered without a header line before it. It"
|
149
160
|
output.puts "could be that logging is not setup correctly for your application."
|
150
161
|
output.puts "Visit this website for logging configuration tips:"
|
151
162
|
output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
|
@@ -27,13 +27,12 @@ module RequestLogAnalyzer
|
|
27
27
|
options[:database] = arguments[:database]
|
28
28
|
options[:reset_database] = arguments[:reset_database]
|
29
29
|
options[:debug] = arguments[:debug]
|
30
|
-
options[:yaml] = arguments[:dump]
|
30
|
+
options[:yaml] = arguments[:yaml] || arguments[:dump]
|
31
31
|
options[:mail] = arguments[:mail]
|
32
32
|
options[:no_progress] = arguments[:no_progress]
|
33
33
|
options[:format] = arguments[:format]
|
34
|
-
options[:output] = arguments[:output]
|
34
|
+
options[:output] = arguments[:output].downcase
|
35
35
|
options[:file] = arguments[:file]
|
36
|
-
options[:format] = arguments[:format]
|
37
36
|
options[:after] = arguments[:after]
|
38
37
|
options[:before] = arguments[:before]
|
39
38
|
options[:reject] = arguments[:reject]
|
@@ -43,6 +42,7 @@ module RequestLogAnalyzer
|
|
43
42
|
options[:report_width] = arguments[:report_width]
|
44
43
|
options[:report_sort] = arguments[:report_sort]
|
45
44
|
options[:report_amount] = arguments[:report_amount]
|
45
|
+
options[:mailhost] = arguments[:mailhost]
|
46
46
|
|
47
47
|
# Apache format workaround
|
48
48
|
if arguments[:rails_format]
|
@@ -51,6 +51,12 @@ module RequestLogAnalyzer
|
|
51
51
|
options[:format] = {:apache => arguments[:apache_format]}
|
52
52
|
end
|
53
53
|
|
54
|
+
# Handle output format casing
|
55
|
+
if options[:output].class == String
|
56
|
+
options[:output] = 'HTML' if options[:output] == 'html'
|
57
|
+
options[:output] = 'FixedWidth' if options[:output] == 'fixedwidth' || options[:output] == 'fixed_width'
|
58
|
+
end
|
59
|
+
|
54
60
|
# Register sources
|
55
61
|
if arguments.parameters.length == 1
|
56
62
|
file = arguments.parameters[0]
|
@@ -66,6 +72,20 @@ module RequestLogAnalyzer
|
|
66
72
|
options.store(:source_files, arguments.parameters)
|
67
73
|
end
|
68
74
|
|
75
|
+
# Guess file format
|
76
|
+
if !options[:format] && options[:source_files]
|
77
|
+
options[:format] = :rails # Default
|
78
|
+
|
79
|
+
if options[:source_files] != $stdin
|
80
|
+
if options[:source_files].class == String
|
81
|
+
options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files])
|
82
|
+
|
83
|
+
elsif options[:source_files].class == Array && options[:source_files].first != $stdin
|
84
|
+
options[:format] = RequestLogAnalyzer::FileFormat.autodetect(options[:source_files].first)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
69
89
|
build(options)
|
70
90
|
end
|
71
91
|
|
@@ -80,12 +100,13 @@ module RequestLogAnalyzer
|
|
80
100
|
# * <tt>:database</tt> Database file to insert encountered requests to.
|
81
101
|
# * <tt>:debug</tt> Enables echo aggregator which will echo each request analyzed.
|
82
102
|
# * <tt>:file</tt> Filestring, File or StringIO.
|
83
|
-
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb,
|
103
|
+
# * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
|
84
104
|
# * <tt>:mail</tt> Email the results to this email address.
|
85
|
-
# * <tt>:
|
86
|
-
# * <tt>:
|
105
|
+
# * <tt>:mailhost</tt> Email the results to this mail server.
|
106
|
+
# * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
|
107
|
+
# * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
|
87
108
|
# * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
|
88
|
-
# * <tt>:report_width</tt> Width
|
109
|
+
# * <tt>:report_width</tt> Width of reports in characters for FixedWidth reports. (Defaults to 80)
|
89
110
|
# * <tt>:reset_database</tt> Reset the database before starting.
|
90
111
|
# * <tt>:select</tt> Select specific {:field => :value} combination (expects a single hash).
|
91
112
|
# * <tt>:source_files</tt> Source files to analyze. Provide either File, array of files or STDIN.
|
@@ -104,7 +125,7 @@ module RequestLogAnalyzer
|
|
104
125
|
# * Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
|
105
126
|
def self.build(options)
|
106
127
|
# Defaults
|
107
|
-
options[:output] ||= '
|
128
|
+
options[:output] ||= 'FixedWidth'
|
108
129
|
options[:format] ||= :rails
|
109
130
|
options[:aggregator] ||= [:summarizer]
|
110
131
|
options[:report_width] ||= 80
|
@@ -113,7 +134,7 @@ module RequestLogAnalyzer
|
|
113
134
|
options[:boring] ||= false
|
114
135
|
|
115
136
|
# Deprecation warnings
|
116
|
-
if options[:dump]
|
137
|
+
if options[:dump]
|
117
138
|
warn "[DEPRECATION] `:dump` is deprecated. Please use `:yaml` instead."
|
118
139
|
options[:yaml] = options[:dump]
|
119
140
|
end
|
@@ -134,7 +155,7 @@ module RequestLogAnalyzer
|
|
134
155
|
output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
|
135
156
|
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
136
157
|
elsif options[:mail]
|
137
|
-
output_object = RequestLogAnalyzer::Mailer.new(options[:mail])
|
158
|
+
output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
|
138
159
|
output_args = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
|
139
160
|
else
|
140
161
|
output_object = STDOUT
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
2
|
|
3
|
-
def self.const_missing(const)
|
3
|
+
def self.const_missing(const) # :nodoc:
|
4
4
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
5
5
|
end
|
6
6
|
|
@@ -39,10 +39,54 @@ module RequestLogAnalyzer::FileFormat
|
|
39
39
|
|
40
40
|
# check the returned klass to see if it can be used
|
41
41
|
raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
|
42
|
-
raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
42
|
+
raise "Invalid FileFormat class from #{file_format.inspect}" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
|
43
43
|
|
44
44
|
@current_file_format = klass.create(*args) # return an instance of the class
|
45
45
|
end
|
46
|
+
|
47
|
+
# Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
|
48
|
+
def self.all_formats
|
49
|
+
@all_formats ||= Dir[File.dirname(__FILE__) + '/file_format/*.rb'].map do |file|
|
50
|
+
self.load(File.basename(file, '.rb'))
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Autodetects the filetype of a given file.
|
55
|
+
#
|
56
|
+
# Returns a FileFormat instance, by parsing the first couple of lines of the provided file
|
57
|
+
# with avery known file format and return the most promosing file format based on the parser
|
58
|
+
# statistics. The <tt>autodetect_score</tt> method is used to score the fitness of a format.
|
59
|
+
#
|
60
|
+
# <tt>file</tt>:: The file to detect the file format for.
|
61
|
+
# <tt>line_count</tt>:: The number of lines to take into consideration
|
62
|
+
def self.autodetect(file, line_count = 50)
|
63
|
+
|
64
|
+
parsers = all_formats.map { |f| RequestLogAnalyzer::Source::LogParser.new(f, :parse_strategy => 'cautious') }
|
65
|
+
|
66
|
+
File.open(file, 'r') do |io|
|
67
|
+
while io.lineno < line_count && (line = io.gets)
|
68
|
+
parsers.each { |parser| parser.parse_line(line) }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
parsers.select { |p| autodetect_score(p) > 0 }.max { |a, b| autodetect_score(a) <=> autodetect_score(b) }.file_format rescue nil
|
73
|
+
end
|
74
|
+
|
75
|
+
# Calculates a file format auto detection score based on the parser statistics.
|
76
|
+
#
|
77
|
+
# This method returns a score as an integer. Usually, the score will increase as more
|
78
|
+
# lines are parsed. Usually, a file_format with a score of zero or lower should not be
|
79
|
+
# considered.
|
80
|
+
#
|
81
|
+
# <tt>parser</tt>:: The parsed that was use to parse the initial lines of the log file.
|
82
|
+
def self.autodetect_score(parser)
|
83
|
+
score = 0
|
84
|
+
score -= parser.file_format.line_definitions.length
|
85
|
+
score -= parser.warnings * 3
|
86
|
+
score += parser.parsed_lines * 1
|
87
|
+
score += parser.parsed_requests * 10
|
88
|
+
score
|
89
|
+
end
|
46
90
|
|
47
91
|
# Base class for all log file format definitions. This class provides functions for subclasses to
|
48
92
|
# define their LineDefinitions and to define a summary report.
|
@@ -55,7 +55,7 @@ module RequestLogAnalyzer::FileFormat
|
|
55
55
|
|
56
56
|
# Creates the access log line definition based on the Apache log format string
|
57
57
|
def self.access_line_definition(format_string)
|
58
|
-
format_string ||= :
|
58
|
+
format_string ||= :common
|
59
59
|
format_string = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
|
60
60
|
|
61
61
|
line_regexp = ''
|
@@ -0,0 +1,93 @@
|
|
1
|
+
module RequestLogAnalyzer::FileFormat
|
2
|
+
|
3
|
+
class Mysql < Base
|
4
|
+
|
5
|
+
line_definition :time do |line|
|
6
|
+
line.header = :alternative
|
7
|
+
line.teaser = /\# Time: /
|
8
|
+
line.regexp = /\# Time: (\d{6}\s+\d{1,2}:\d{2}:\d{2})/
|
9
|
+
line.captures << { :name => :timestamp, :type => :timestamp }
|
10
|
+
end
|
11
|
+
|
12
|
+
line_definition :user_host do |line|
|
13
|
+
line.header = :alternative
|
14
|
+
line.teaser = /\# User\@Host\: /
|
15
|
+
line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[([\d\.]*)\]/
|
16
|
+
line.captures << { :name => :user, :type => :string } <<
|
17
|
+
{ :name => :host, :type => :string } <<
|
18
|
+
{ :name => :ip, :type => :string }
|
19
|
+
end
|
20
|
+
|
21
|
+
line_definition :query_statistics do |line|
|
22
|
+
line.header = :alternative
|
23
|
+
line.teaser = /\# Query_time: /
|
24
|
+
line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/
|
25
|
+
line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
|
26
|
+
{ :name => :lock_time, :type => :duration, :unit => :sec } <<
|
27
|
+
{ :name => :rows_sent, :type => :integer } <<
|
28
|
+
{ :name => :rows_examined, :type => :integer }
|
29
|
+
end
|
30
|
+
|
31
|
+
line_definition :use_database do |line|
|
32
|
+
line.regexp = /^use (\w+);\s*$/
|
33
|
+
line.captures << { :name => :database, :type => :string }
|
34
|
+
end
|
35
|
+
|
36
|
+
line_definition :query_part do |line|
|
37
|
+
line.regexp = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
|
38
|
+
line.captures << { :name => :query_fragment, :type => :string }
|
39
|
+
end
|
40
|
+
|
41
|
+
line_definition :query do |line|
|
42
|
+
line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
|
43
|
+
line.captures << { :name => :query, :type => :sql }
|
44
|
+
line.footer = true
|
45
|
+
end
|
46
|
+
|
47
|
+
PER_USER = :user
|
48
|
+
PER_QUERY = :query
|
49
|
+
PER_USER_QUERY = Proc.new { |request| "#{request[:user]}@#{request.host}: #{request[:query]}" }
|
50
|
+
|
51
|
+
report do |analyze|
|
52
|
+
analyze.timespan :line_type => :time
|
53
|
+
analyze.frequency :user, :title => "Users with most queries"
|
54
|
+
analyze.duration :query_time, :category => PER_USER, :title => 'Query time per user'
|
55
|
+
analyze.duration :query_time, :category => PER_USER_QUERY, :title => 'Query time'
|
56
|
+
# => analyze.duration :lock_time, :category => PER_USER_QUERY, :title => 'Lock time'
|
57
|
+
analyze.count :category => PER_USER_QUERY, :title => "Rows examined", :field => :rows_examined
|
58
|
+
analyze.count :category => PER_USER_QUERY, :title => "Rows sent", :field => :rows_sent
|
59
|
+
end
|
60
|
+
|
61
|
+
class Request < RequestLogAnalyzer::Request
|
62
|
+
|
63
|
+
def convert_sql(value, definition)
|
64
|
+
|
65
|
+
# Recreate the full SQL query by joining all the previous parts and this last line
|
66
|
+
sql = every(:query_fragment).join("\n") + value
|
67
|
+
|
68
|
+
# Sanitize an SQL query so that it can be used as a category field.
|
69
|
+
# sql.gsub!(/\/\*.*\*\//, '') # remove comments
|
70
|
+
sql.gsub!(/\s+/, ' ') # remove excessive whitespace
|
71
|
+
sql.gsub!(/`([^`]+)`/, '\1') # remove quotes from field names
|
72
|
+
sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date') # replace dates
|
73
|
+
sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime') # replace timestamps
|
74
|
+
sql.gsub!(/'[^']*'/, ':string') # replace strings
|
75
|
+
sql.gsub!(/\b\d+\b/, ':int') # replace integers
|
76
|
+
sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list
|
77
|
+
sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list
|
78
|
+
|
79
|
+
return sql.rstrip
|
80
|
+
end
|
81
|
+
|
82
|
+
def host
|
83
|
+
self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
|
84
|
+
end
|
85
|
+
|
86
|
+
# Convert the timestamp to an integer
|
87
|
+
def convert_timestamp(value, definition)
|
88
|
+
all,y,m,d,h,i,s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
89
|
+
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -3,7 +3,7 @@ module RequestLogAnalyzer
|
|
3
3
|
# Mail report to a specified emailaddress
|
4
4
|
class Mailer
|
5
5
|
|
6
|
-
attr_accessor :data, :to, :host
|
6
|
+
attr_accessor :data, :to, :host, :port
|
7
7
|
|
8
8
|
# Initialize a mailer
|
9
9
|
# <tt>to</tt> to email address to mail to
|
@@ -19,8 +19,11 @@ module RequestLogAnalyzer
|
|
19
19
|
require 'net/smtp'
|
20
20
|
@to = to
|
21
21
|
@host = host
|
22
|
+
|
23
|
+
@port = 25
|
22
24
|
@options = options
|
23
|
-
@
|
25
|
+
@host, @port = host.split(':') if @host.include?(':')
|
26
|
+
@data = []
|
24
27
|
end
|
25
28
|
|
26
29
|
# Send all data in @data to the email address used during initialization.
|
@@ -38,11 +41,11 @@ To: #{to_alias} <#{@to}>
|
|
38
41
|
Subject: #{subject}
|
39
42
|
#{content_type}
|
40
43
|
|
41
|
-
#{@data.
|
44
|
+
#{@data.join("\n")}
|
42
45
|
END_OF_MESSAGE
|
43
46
|
|
44
47
|
unless @options[:debug]
|
45
|
-
Net::SMTP.start(@host) do |smtp|
|
48
|
+
Net::SMTP.start(@host, @port) do |smtp|
|
46
49
|
smtp.send_message msg, from, to
|
47
50
|
end
|
48
51
|
end
|
@@ -65,6 +65,11 @@ module RequestLogAnalyzer
|
|
65
65
|
else raise "Unknown duration unit"
|
66
66
|
end
|
67
67
|
end
|
68
|
+
|
69
|
+
# Convert an epoch to an integer
|
70
|
+
def convert_epoch(value, capture_definition)
|
71
|
+
Time.at(value.to_i).strftime('%Y%m%d%H%M%S').to_i
|
72
|
+
end
|
68
73
|
end
|
69
74
|
|
70
75
|
# Install the default converter methods
|
@@ -20,6 +20,7 @@ module RequestLogAnalyzer::Source
|
|
20
20
|
PARSE_STRATEGIES = ['cautious', 'assume-correct']
|
21
21
|
|
22
22
|
attr_reader :source_files, :current_file, :current_lineno
|
23
|
+
attr_reader :warnings, :parsed_lines, :parsed_requests, :skipped_lines, :skipped_requests
|
23
24
|
|
24
25
|
# Initializes the log file parser instance.
|
25
26
|
# It will apply the language specific FileFormat module to this instance. It will use the line
|
@@ -29,6 +30,7 @@ module RequestLogAnalyzer::Source
|
|
29
30
|
# <tt>options</tt>:: A hash of options that are used by the parser
|
30
31
|
def initialize(format, options = {})
|
31
32
|
super(format, options)
|
33
|
+
@warnings = 0
|
32
34
|
@parsed_lines = 0
|
33
35
|
@parsed_requests = 0
|
34
36
|
@skipped_lines = 0
|
@@ -142,21 +144,27 @@ module RequestLogAnalyzer::Source
|
|
142
144
|
# <tt>io</tt>:: The IO instance to use as source
|
143
145
|
# <tt>options</tt>:: A hash of options that can be used by the parser.
|
144
146
|
def parse_io(io, options = {}, &block) # :yields: request
|
145
|
-
@current_lineno =
|
147
|
+
@current_lineno = 0
|
146
148
|
while line = io.gets
|
147
|
-
@progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
|
148
|
-
|
149
|
-
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
150
|
-
@parsed_lines += 1
|
151
|
-
update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
|
152
|
-
end
|
153
|
-
|
154
149
|
@current_lineno += 1
|
150
|
+
@progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
|
151
|
+
parse_line(line, &block)
|
155
152
|
end
|
156
153
|
|
157
154
|
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
158
155
|
@current_lineno = nil
|
159
156
|
end
|
157
|
+
|
158
|
+
# Parses a single line using the current file format. If successful, use the parsed
|
159
|
+
# information to build a request
|
160
|
+
# <tt>line</tt>:: The line to parse
|
161
|
+
# <tt>block</tt>:: The block to send fully parsed requests to.
|
162
|
+
def parse_line(line, &block) # :yields: request
|
163
|
+
if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
|
164
|
+
@parsed_lines += 1
|
165
|
+
update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
|
166
|
+
end
|
167
|
+
end
|
160
168
|
|
161
169
|
# Add a block to this method to install a progress handler while parsing.
|
162
170
|
# <tt>proc</tt>:: The proc that will be called to handle progress update messages
|
@@ -186,6 +194,7 @@ module RequestLogAnalyzer::Source
|
|
186
194
|
# <tt>type</tt>:: The warning type (a Symbol)
|
187
195
|
# <tt>message</tt>:: A message explaining the warning
|
188
196
|
def warn(type, message)
|
197
|
+
@warnings += 1
|
189
198
|
@warning_handler.call(type, message, @current_lineno) if @warning_handler
|
190
199
|
end
|
191
200
|
|
@@ -214,7 +223,13 @@ module RequestLogAnalyzer::Source
|
|
214
223
|
#
|
215
224
|
# <tt>request_data</tt>:: A hash of data that was parsed from the last line.
|
216
225
|
def update_current_request(request_data, &block) # :yields: request
|
217
|
-
if
|
226
|
+
if alternative_header_line?(request_data)
|
227
|
+
if @current_request
|
228
|
+
@current_request << request_data
|
229
|
+
else
|
230
|
+
@current_request = @file_format.request(request_data)
|
231
|
+
end
|
232
|
+
elsif header_line?(request_data)
|
218
233
|
if @current_request
|
219
234
|
case options[:parse_strategy]
|
220
235
|
when 'assume-correct'
|
@@ -258,10 +273,18 @@ module RequestLogAnalyzer::Source
|
|
258
273
|
@skipped_requests += 1 unless accepted
|
259
274
|
end
|
260
275
|
|
276
|
+
|
277
|
+
# Checks whether a given line hash is an alternative header line according to the current file format.
|
278
|
+
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
279
|
+
def alternative_header_line?(hash)
|
280
|
+
hash[:line_definition].header == :alternative
|
281
|
+
end
|
282
|
+
|
283
|
+
|
261
284
|
# Checks whether a given line hash is a header line according to the current file format.
|
262
285
|
# <tt>hash</tt>:: A hash of data that was parsed from the line.
|
263
286
|
def header_line?(hash)
|
264
|
-
hash[:line_definition].header
|
287
|
+
hash[:line_definition].header == true
|
265
288
|
end
|
266
289
|
|
267
290
|
# Checks whether a given line hash is a footer line according to the current file format.
|