request-log-analyzer 1.10.1 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +0 -1
- data/lib/request_log_analyzer.rb +15 -29
- data/lib/request_log_analyzer/aggregator.rb +5 -5
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +2 -1
- data/lib/request_log_analyzer/controller.rb +0 -3
- data/lib/request_log_analyzer/database.rb +6 -7
- data/lib/request_log_analyzer/file_format.rb +42 -13
- data/lib/request_log_analyzer/file_format/apache.rb +1 -1
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +2 -2
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +2 -2
- data/lib/request_log_analyzer/file_format/haproxy.rb +107 -13
- data/lib/request_log_analyzer/file_format/mysql.rb +5 -5
- data/lib/request_log_analyzer/file_format/rails3.rb +7 -0
- data/lib/request_log_analyzer/filter.rb +4 -5
- data/lib/request_log_analyzer/line_definition.rb +6 -4
- data/lib/request_log_analyzer/output.rb +3 -5
- data/lib/request_log_analyzer/source.rb +3 -4
- data/lib/request_log_analyzer/source/log_parser.rb +56 -4
- data/lib/request_log_analyzer/tracker.rb +8 -8
- data/request-log-analyzer.gemspec +3 -3
- data/spec/fixtures/mysql_slow_query.log +0 -1
- data/spec/integration/command_line_usage_spec.rb +0 -5
- data/spec/lib/helpers.rb +2 -2
- data/spec/lib/matchers.rb +38 -7
- data/spec/lib/mocks.rb +1 -5
- data/spec/unit/database/base_class_spec.rb +1 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +58 -55
- data/spec/unit/file_format/apache_format_spec.rb +74 -162
- data/spec/unit/file_format/common_regular_expressions_spec.rb +51 -26
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -31
- data/spec/unit/file_format/delayed_job2_format_spec.rb +27 -32
- data/spec/unit/file_format/delayed_job_format_spec.rb +44 -63
- data/spec/unit/file_format/haproxy_format_spec.rb +69 -71
- data/spec/unit/file_format/line_definition_spec.rb +26 -33
- data/spec/unit/file_format/merb_format_spec.rb +22 -37
- data/spec/unit/file_format/mysql_format_spec.rb +80 -123
- data/spec/unit/file_format/oink_format_spec.rb +29 -61
- data/spec/unit/file_format/postgresql_format_spec.rb +2 -4
- data/spec/unit/file_format/rack_format_spec.rb +49 -44
- data/spec/unit/file_format/rails3_format_spec.rb +17 -20
- data/spec/unit/file_format/rails_format_spec.rb +52 -68
- data/spec/unit/file_format/w3c_format_spec.rb +40 -39
- data/spec/unit/source/log_parser_spec.rb +1 -1
- metadata +4 -7
- data/lib/mixins/gets_memory_protection.rb +0 -80
- data/lib/request_log_analyzer/output/fancy_html.rb +0 -44
- data/lib/request_log_analyzer/source/database_loader.rb +0 -87
@@ -11,7 +11,7 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
it "should have a valid language" do
|
14
|
-
@log_parser.file_format.should
|
14
|
+
@log_parser.file_format.should be_well_formed
|
15
15
|
end
|
16
16
|
|
17
17
|
it "should set the :source for every parsed line" do
|
metadata
CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
|
|
4
4
|
prerelease: false
|
5
5
|
segments:
|
6
6
|
- 1
|
7
|
-
-
|
8
|
-
-
|
9
|
-
version: 1.
|
7
|
+
- 11
|
8
|
+
- 0
|
9
|
+
version: 1.11.0
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Willem van Bergen
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-
|
18
|
+
date: 2011-04-27 00:00:00 -04:00
|
19
19
|
default_executable: request-log-analyzer
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|
@@ -95,7 +95,6 @@ files:
|
|
95
95
|
- lib/cli/database_console_init.rb
|
96
96
|
- lib/cli/progressbar.rb
|
97
97
|
- lib/cli/tools.rb
|
98
|
-
- lib/mixins/gets_memory_protection.rb
|
99
98
|
- lib/request_log_analyzer.rb
|
100
99
|
- lib/request_log_analyzer/aggregator.rb
|
101
100
|
- lib/request_log_analyzer/aggregator/database_inserter.rb
|
@@ -132,12 +131,10 @@ files:
|
|
132
131
|
- lib/request_log_analyzer/log_processor.rb
|
133
132
|
- lib/request_log_analyzer/mailer.rb
|
134
133
|
- lib/request_log_analyzer/output.rb
|
135
|
-
- lib/request_log_analyzer/output/fancy_html.rb
|
136
134
|
- lib/request_log_analyzer/output/fixed_width.rb
|
137
135
|
- lib/request_log_analyzer/output/html.rb
|
138
136
|
- lib/request_log_analyzer/request.rb
|
139
137
|
- lib/request_log_analyzer/source.rb
|
140
|
-
- lib/request_log_analyzer/source/database_loader.rb
|
141
138
|
- lib/request_log_analyzer/source/log_parser.rb
|
142
139
|
- lib/request_log_analyzer/tracker.rb
|
143
140
|
- lib/request_log_analyzer/tracker/duration.rb
|
@@ -1,80 +0,0 @@
|
|
1
|
-
class File
|
2
|
-
alias_method :gets_original, :gets
|
3
|
-
# The size of the reads we will use to add to the line buffer.
|
4
|
-
MAX_READ_SIZE=1024*100
|
5
|
-
|
6
|
-
#
|
7
|
-
# This method returns the next line of the File.
|
8
|
-
#
|
9
|
-
# It works by moving the file pointer forward +MAX_READ_SIZE+ at a time,
|
10
|
-
# storing seen lines in <tt>@line_buffer</tt>. Once the buffer contains at
|
11
|
-
# least two lines (ensuring we have seen on full line) or the file pointer
|
12
|
-
# reaches the end of the File, the last line from the buffer is returned.
|
13
|
-
# When the buffer is exhausted, this will throw +nil+ (from the empty Array).
|
14
|
-
#
|
15
|
-
# Read portions of the file that do not contain the +sep_string+ are not added to
|
16
|
-
# the buffer. This prevents <tt>@line_buffer<tt> from growing signficantly when parsing
|
17
|
-
# large lines.
|
18
|
-
#
|
19
|
-
def gets(sep_string = $/)
|
20
|
-
@read_size ||= MAX_READ_SIZE
|
21
|
-
# A buffer to hold lines read, but not yet returned.
|
22
|
-
@line_buffer ||= Array.new
|
23
|
-
|
24
|
-
# Record where we are.
|
25
|
-
@current_pos ||= pos
|
26
|
-
|
27
|
-
# Last Position in the file
|
28
|
-
@last_pos ||= nil
|
29
|
-
if @last_pos.nil?
|
30
|
-
seek(0, IO::SEEK_END)
|
31
|
-
@last_pos = pos
|
32
|
-
seek(@current_pos, IO::SEEK_SET)
|
33
|
-
end
|
34
|
-
|
35
|
-
#
|
36
|
-
# If we have more than one line in the buffer or we have reached the
|
37
|
-
# beginning of the file, send the last line in the buffer to the caller.
|
38
|
-
# (This may be +nil+, if the buffer has been exhausted.)
|
39
|
-
#
|
40
|
-
if @line_buffer.size > 2 or @current_pos >= @last_pos
|
41
|
-
self.lineno += 1
|
42
|
-
return @line_buffer.shift
|
43
|
-
end
|
44
|
-
|
45
|
-
sep =
|
46
|
-
|
47
|
-
chunk = String.new
|
48
|
-
while chunk and chunk !~ /#{sep_string}/
|
49
|
-
chunk = read(@read_size)
|
50
|
-
end
|
51
|
-
|
52
|
-
# Appends new lines to the last element of the buffer
|
53
|
-
line_buffer_pos = @line_buffer.any? ? @line_buffer.size-1 : 0
|
54
|
-
|
55
|
-
if chunk
|
56
|
-
@line_buffer[line_buffer_pos] = @line_buffer[line_buffer_pos].to_s<< chunk
|
57
|
-
else
|
58
|
-
# at the end
|
59
|
-
return @line_buffer.shift
|
60
|
-
end
|
61
|
-
|
62
|
-
#
|
63
|
-
# Divide the last line of the buffer based on +sep_string+ and #flatten!
|
64
|
-
# those new lines into the buffer.
|
65
|
-
#
|
66
|
-
@line_buffer[line_buffer_pos] = @line_buffer[line_buffer_pos].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
|
67
|
-
@line_buffer.flatten!
|
68
|
-
|
69
|
-
#
|
70
|
-
# If we made it this far, we need to read more data to try and find the
|
71
|
-
# end of a line or the end of the file. Move the file pointer
|
72
|
-
# forward a step, to give us new bytes to read.
|
73
|
-
#
|
74
|
-
@current_pos += @read_size
|
75
|
-
seek(@current_pos, IO::SEEK_SET)
|
76
|
-
|
77
|
-
# We have more data now, so try again to read a line...
|
78
|
-
gets(sep_string)
|
79
|
-
end
|
80
|
-
end
|
@@ -1,44 +0,0 @@
|
|
1
|
-
begin
|
2
|
-
require 'rubygems'
|
3
|
-
require 'gchart'
|
4
|
-
rescue LoadError
|
5
|
-
$stderr.puts "The FancyHTML output format requires the googlechart gem:"
|
6
|
-
$stderr.puts " (sudo) gem install googlecharts"
|
7
|
-
end
|
8
|
-
|
9
|
-
module RequestLogAnalyzer::Output
|
10
|
-
|
11
|
-
class FancyHTML < HTML
|
12
|
-
|
13
|
-
def report_tracker(tracker)
|
14
|
-
case tracker
|
15
|
-
when RequestLogAnalyzer::Tracker::HourlySpread then report_hourly_spread(tracker)
|
16
|
-
else tracker.report(self)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def report_hourly_spread(tracker)
|
21
|
-
title tracker.title
|
22
|
-
puts tag(:img, nil, :width => '700', :height => '120', :src =>
|
23
|
-
Gchart.sparkline(:data => tracker.hour_frequencies, :size => '700x120', :line_colors => '0077CC',
|
24
|
-
:axis_with_labels => 'x,y', :axis_labels => [x_axis_labels(tracker),y_axis_labels(tracker)]))
|
25
|
-
end
|
26
|
-
|
27
|
-
def x_axis_labels(tracker)
|
28
|
-
frmt = '%H:%M'
|
29
|
-
x_axis_labels = []
|
30
|
-
num_labels = 5
|
31
|
-
start = tracker.first_timestamp
|
32
|
-
step = tracker.timespan / (num_labels - 1)
|
33
|
-
for i in 0...num_labels do
|
34
|
-
x_axis_labels << (start + step * i).strftime(frmt)
|
35
|
-
end
|
36
|
-
x_axis_labels
|
37
|
-
end
|
38
|
-
|
39
|
-
def y_axis_labels(tracker)
|
40
|
-
sorted_frequencies = tracker.hour_frequencies.sort
|
41
|
-
[sorted_frequencies.first, sorted_frequencies.last]
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
@@ -1,87 +0,0 @@
|
|
1
|
-
require 'rubygems'
|
2
|
-
require 'activerecord'
|
3
|
-
|
4
|
-
module RequestLogAnalyzer::Source
|
5
|
-
|
6
|
-
# Active Resource hook
|
7
|
-
class Request < ActiveRecord::Base
|
8
|
-
has_many :completed_lines
|
9
|
-
has_many :processing_lines
|
10
|
-
def convert(file_format)
|
11
|
-
send_attributes = self.attributes
|
12
|
-
send_attributes.merge!(self.completed_lines.first.attributes) if self.completed_lines.first
|
13
|
-
send_attributes.merge!(self.processing_lines.first.attributes) if self.processing_lines.first
|
14
|
-
return RequestLogAnalyzer::Request.new(file_format, send_attributes)
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
class CompletedLine < ActiveRecord::Base
|
19
|
-
belongs_to :request
|
20
|
-
end
|
21
|
-
|
22
|
-
class ProcessingLine < ActiveRecord::Base
|
23
|
-
belongs_to :request
|
24
|
-
end
|
25
|
-
|
26
|
-
# The Database class gets log data from the database.
|
27
|
-
class DatabaseLoader < Base
|
28
|
-
|
29
|
-
attr_reader :source_files, :file_format, :requests
|
30
|
-
|
31
|
-
# Initializes the log file parser instance.
|
32
|
-
# It will apply the language specific FileFormat module to this instance. It will use the line
|
33
|
-
# definitions in this module to parse any input that it is given (see parse_io).
|
34
|
-
#
|
35
|
-
# <tt>format</tt>:: The current file format instance
|
36
|
-
# <tt>options</tt>:: A hash of options that are used by the parser
|
37
|
-
def initialize(format, options = {})
|
38
|
-
super(format, options)
|
39
|
-
@source_files = options[:source_files]
|
40
|
-
@parsed_requests = 0
|
41
|
-
@requests = []
|
42
|
-
end
|
43
|
-
|
44
|
-
# Reads the input, which can either be a file, sequence of files or STDIN to parse
|
45
|
-
# lines specified in the FileFormat. This lines will be combined into Request instances,
|
46
|
-
# that will be yielded. The actual parsing occurs in the parse_io method.
|
47
|
-
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
48
|
-
def each_request(options = {}, &block) # :yields: request
|
49
|
-
ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => @source_files)
|
50
|
-
|
51
|
-
@progress_handler.call(:started, @source_files) if @progress_handler
|
52
|
-
RequestLogAnalyzer::Source::Request.find(:all).each do |request|
|
53
|
-
@parsed_requests += 1
|
54
|
-
@progress_handler.call(:progress, @parsed_requests) if @progress_handler
|
55
|
-
|
56
|
-
yield request.convert(self.file_format)
|
57
|
-
end
|
58
|
-
|
59
|
-
@progress_handler.call(:finished, @source_files) if @progress_handler
|
60
|
-
end
|
61
|
-
|
62
|
-
# Add a block to this method to install a progress handler while parsing.
|
63
|
-
# <tt>proc</tt>:: The proc that will be called to handle progress update messages
|
64
|
-
def progress=(proc)
|
65
|
-
@progress_handler = proc
|
66
|
-
end
|
67
|
-
|
68
|
-
# Add a block to this method to install a warning handler while parsing,
|
69
|
-
# <tt>proc</tt>:: The proc that will be called to handle parse warning messages
|
70
|
-
def warning=(proc)
|
71
|
-
@warning_handler = proc
|
72
|
-
end
|
73
|
-
|
74
|
-
# This method is called by the parser if it encounteres any parsing problems.
|
75
|
-
# It will call the installed warning handler if any.
|
76
|
-
#
|
77
|
-
# By default, RequestLogAnalyzer::Controller will install a warning handler
|
78
|
-
# that will pass the warnings to each aggregator so they can do something useful
|
79
|
-
# with it.
|
80
|
-
#
|
81
|
-
# <tt>type</tt>:: The warning type (a Symbol)
|
82
|
-
# <tt>message</tt>:: A message explaining the warning
|
83
|
-
def warn(type, message)
|
84
|
-
@warning_handler.call(type, message, @current_io.lineno) if @warning_handler
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|