request-log-analyzer 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +1 -1
- data/README.rdoc +19 -24
- data/bin/request-log-analyzer +10 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +13 -2
- data/lib/request_log_analyzer/controller.rb +68 -36
- data/lib/request_log_analyzer/file_format/apache.rb +1 -1
- data/lib/request_log_analyzer/file_format/mysql.rb +93 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +1 -1
- data/lib/request_log_analyzer/file_format.rb +46 -2
- data/lib/request_log_analyzer/mailer.rb +25 -12
- data/lib/request_log_analyzer/output/fixed_width.rb +1 -1
- data/lib/request_log_analyzer/output/html.rb +2 -0
- data/lib/request_log_analyzer/request.rb +5 -0
- data/lib/request_log_analyzer/source/log_parser.rb +33 -10
- data/lib/request_log_analyzer/tracker/count.rb +93 -0
- data/lib/request_log_analyzer.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/fixtures/mysql_slow_query.log +110 -0
- data/spec/integration/command_line_usage_spec.rb +3 -3
- data/spec/integration/mailer_spec.rb +179 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +152 -0
- data/spec/lib/helpers.rb +26 -1
- data/spec/lib/mocks.rb +3 -1
- data/spec/unit/file_format/apache_format_spec.rb +102 -95
- data/spec/unit/file_format/format_autodetection_spec.rb +35 -0
- data/spec/unit/file_format/mysql_format_spec.rb +155 -0
- data/spec/unit/file_format/rails_format_spec.rb +9 -12
- data/spec/unit/mailer_spec.rb +12 -0
- data/spec/unit/tracker/count_tracker_spec.rb +123 -0
- data/spec/unit/tracker/tracker_api_spec.rb +1 -1
- metadata +125 -110
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper.rb'
|
|
2
|
+
|
|
3
|
+
def capture_stdout_and_stderr_with_warnings_on
|
|
4
|
+
$stdout, $stderr, warnings, $VERBOSE =
|
|
5
|
+
StringIO.new, StringIO.new, $VERBOSE, true
|
|
6
|
+
yield
|
|
7
|
+
return $stdout.string, $stderr.string
|
|
8
|
+
ensure
|
|
9
|
+
$stdout, $stderr, $VERBOSE = STDOUT, STDERR, warnings
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
describe RequestLogAnalyzer, 'when using the rla API like the scout plugin' do
|
|
13
|
+
|
|
14
|
+
before(:each) do
|
|
15
|
+
# prepare a place to capture the output
|
|
16
|
+
sio = StringIO.new
|
|
17
|
+
|
|
18
|
+
# place an IO object where I want RequestLogAnalyzer to read from
|
|
19
|
+
open(log_fixture(:rails_1x)) do |log|
|
|
20
|
+
completed_count = 0
|
|
21
|
+
log.each do |line|
|
|
22
|
+
completed_count += 1 if line =~ /\ACompleted\b/
|
|
23
|
+
break if completed_count == 2 # skipping first two requests
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# trigger the log parse
|
|
27
|
+
@stdout, @stderr = capture_stdout_and_stderr_with_warnings_on do
|
|
28
|
+
RequestLogAnalyzer::Controller.build(
|
|
29
|
+
:output => EmbeddedHTML,
|
|
30
|
+
:file => sio,
|
|
31
|
+
:after => Time.local(2008, 8, 14, 21, 16, 31), # after 3rd req
|
|
32
|
+
:source_files => log,
|
|
33
|
+
:format => RequestLogAnalyzer::FileFormat::Rails
|
|
34
|
+
).run!
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# read the resulting output
|
|
39
|
+
@analysis = sio.string
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
it "should generate an analysis" do
|
|
43
|
+
@analysis.should_not be_empty
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
it "should generate customized output using the passed Class" do
|
|
47
|
+
credit = %r{<p>Powered by request-log-analyzer v\d+(?:\.\d+)+</p>\z}
|
|
48
|
+
@analysis.should match(credit)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
it "should skip requests before :after Time" do
|
|
52
|
+
@analysis.should_not include("PeopleController#show")
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it "should include requests after IO#pos and :after Time" do
|
|
56
|
+
@analysis.should include("PeopleController#picture")
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
it "should skip requests before IO#pos" do
|
|
60
|
+
@analysis.should_not include("PeopleController#index")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
it "should not print to $stdout" do
|
|
64
|
+
@stdout.should be_empty
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it "should not print to $stderr (with warnings on)" do
|
|
68
|
+
@stderr.should be_empty
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Helpers
|
|
74
|
+
class EmbeddedHTML < RequestLogAnalyzer::Output::Base
|
|
75
|
+
def print(str)
|
|
76
|
+
@io << str
|
|
77
|
+
end
|
|
78
|
+
alias_method :<<, :print
|
|
79
|
+
|
|
80
|
+
def puts(str = "")
|
|
81
|
+
@io << "#{str}<br/>\n"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def title(title)
|
|
85
|
+
@io.puts(tag(:h2, title))
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def line(*font)
|
|
89
|
+
@io.puts(tag(:hr))
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def link(text, url = nil)
|
|
93
|
+
url = text if url.nil?
|
|
94
|
+
tag(:a, text, :href => url)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def table(*columns, &block)
|
|
98
|
+
rows = Array.new
|
|
99
|
+
yield(rows)
|
|
100
|
+
|
|
101
|
+
@io << tag(:table, :cellspacing => 0) do |content|
|
|
102
|
+
if table_has_header?(columns)
|
|
103
|
+
content << tag(:tr) do
|
|
104
|
+
columns.map { |col| tag(:th, col[:title]) }.join("\n")
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
odd = false
|
|
109
|
+
rows.each do |row|
|
|
110
|
+
odd = !odd
|
|
111
|
+
content << tag(:tr) do
|
|
112
|
+
if odd
|
|
113
|
+
row.map { |cell| tag(:td, cell, :class => "alt") }.join("\n")
|
|
114
|
+
else
|
|
115
|
+
row.map { |cell| tag(:td, cell) }.join("\n")
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def header
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def footer
|
|
126
|
+
@io << tag(:hr) << tag(:p, "Powered by request-log-analyzer v#{RequestLogAnalyzer::VERSION}")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
def tag(tag, content = nil, attributes = nil)
|
|
132
|
+
if block_given?
|
|
133
|
+
attributes = content.nil? ? "" : " " + content.map { |(key, value)| "#{key}=\"#{value}\"" }.join(" ")
|
|
134
|
+
content_string = ""
|
|
135
|
+
content = yield(content_string)
|
|
136
|
+
content = content_string unless content_string.empty?
|
|
137
|
+
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
|
138
|
+
else
|
|
139
|
+
attributes = attributes.nil? ? "" : " " + attributes.map { |(key, value)| "#{key}=\"#{value}\"" }.join(" ")
|
|
140
|
+
if content.nil?
|
|
141
|
+
"<#{tag}#{attributes} />"
|
|
142
|
+
else
|
|
143
|
+
if content.class == Float
|
|
144
|
+
"<#{tag}#{attributes}><div class='color_bar' style=\"width:#{(content*200).floor}px;\"/></#{tag}>"
|
|
145
|
+
else
|
|
146
|
+
"<#{tag}#{attributes}>#{content}</#{tag}>"
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
data/spec/lib/helpers.rb
CHANGED
|
@@ -10,6 +10,11 @@ module RequestLogAnalyzer::Spec::Helpers
|
|
|
10
10
|
File.dirname(__FILE__) + "/../fixtures/#{name}.#{extention}"
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
+
# Creates a log file given some lines
|
|
14
|
+
def log_stream(*lines)
|
|
15
|
+
StringIO.new(lines.join("\n") + "\n")
|
|
16
|
+
end
|
|
17
|
+
|
|
13
18
|
# Request loopback
|
|
14
19
|
def request(fields, format = testing_format)
|
|
15
20
|
if fields.kind_of?(Array)
|
|
@@ -42,6 +47,26 @@ module RequestLogAnalyzer::Spec::Helpers
|
|
|
42
47
|
|
|
43
48
|
# Return a filename that can be used as temporary file in specs
|
|
44
49
|
def temp_output_file(file_type)
|
|
45
|
-
"#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp"
|
|
50
|
+
File.expand_path("#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Check if a given string can be found in the given file
|
|
54
|
+
# Returns the line number if found, nil otherwise
|
|
55
|
+
def find_string_in_file(string, file, options = {})
|
|
56
|
+
return nil unless File::exists?(file)
|
|
57
|
+
|
|
58
|
+
line_counter = 0
|
|
59
|
+
|
|
60
|
+
File.open( file ) do |io|
|
|
61
|
+
io.each {|line|
|
|
62
|
+
line_counter += 1
|
|
63
|
+
line.chomp!
|
|
64
|
+
|
|
65
|
+
p line if options[:debug]
|
|
66
|
+
return line_counter if line.include? string
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
return nil
|
|
46
71
|
end
|
|
47
72
|
end
|
data/spec/lib/mocks.rb
CHANGED
|
@@ -51,146 +51,153 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
|
51
51
|
|
|
52
52
|
context '"Common" access log parsing' do
|
|
53
53
|
before(:all) do
|
|
54
|
-
@file_format
|
|
55
|
-
@log_parser
|
|
56
|
-
@sample_1
|
|
57
|
-
@sample_2
|
|
54
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:apache, :common)
|
|
55
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
|
|
56
|
+
@sample_1 = '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223'
|
|
57
|
+
@sample_2 = '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414'
|
|
58
|
+
@nonsense_sample = 'addasdsasadadssadasd'
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
it "should have a valid language definitions" do
|
|
61
62
|
@file_format.should be_valid
|
|
62
63
|
end
|
|
63
64
|
|
|
64
|
-
it "should parse a valid access log line" do
|
|
65
|
-
@file_format.line_definitions[:access].matches(@sample_1).should be_kind_of(Hash)
|
|
66
|
-
end
|
|
67
|
-
|
|
68
65
|
it "should not parse a valid access log line" do
|
|
69
|
-
@file_format.
|
|
66
|
+
@file_format.should_not parse_line(@nonsense_sample)
|
|
70
67
|
end
|
|
71
68
|
|
|
72
69
|
it "should read the correct values from a valid HTTP/1.0 access log line" do
|
|
73
|
-
@
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
70
|
+
@file_format.should parse_line(@sample_1).as(:access).and_capture(
|
|
71
|
+
:remote_host => '1.129.119.13',
|
|
72
|
+
:remote_logname => nil,
|
|
73
|
+
:user => nil,
|
|
74
|
+
:timestamp => 20090908075409,
|
|
75
|
+
:http_status => 200,
|
|
76
|
+
:http_method => 'GET',
|
|
77
|
+
:http_version => '1.0',
|
|
78
|
+
:bytes_sent => 8223)
|
|
82
79
|
end
|
|
83
80
|
|
|
84
81
|
it "should read the correct values from a valid 200 access log line" do
|
|
85
|
-
@
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
82
|
+
@file_format.should parse_line(@sample_2).as(:access).and_capture(
|
|
83
|
+
:remote_host => '1.82.235.29',
|
|
84
|
+
:remote_logname => nil,
|
|
85
|
+
:user => nil,
|
|
86
|
+
:timestamp => 20090908075405,
|
|
87
|
+
:http_status => 200,
|
|
88
|
+
:http_method => 'GET',
|
|
89
|
+
:http_version => '1.1',
|
|
90
|
+
:bytes_sent => 23414)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
it "should parse 10 request from fixture access log without warnings" do
|
|
94
|
+
request_counter.should_receive(:hit!).exactly(10).times
|
|
95
|
+
@log_parser.should_not_receive(:warn)
|
|
96
|
+
|
|
97
|
+
@log_parser.parse_file(log_fixture(:apache_common)) do |request|
|
|
98
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
|
|
93
99
|
end
|
|
94
100
|
end
|
|
95
|
-
|
|
96
|
-
it "should parse 10 request from fixture access log" do
|
|
97
|
-
counter = mock('counter')
|
|
98
|
-
counter.should_receive(:hit!).exactly(10).times
|
|
99
|
-
@log_parser.parse_file(log_fixture(:apache_common)) { counter.hit! }
|
|
100
|
-
end
|
|
101
101
|
end
|
|
102
102
|
|
|
103
|
-
context '"
|
|
104
|
-
before(:each) do
|
|
105
|
-
@file_format = RequestLogAnalyzer::FileFormat.load(:rack)
|
|
106
|
-
@log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
|
|
107
|
-
@sample_1 = '127.0.0.1 - - [16/Sep/2009 06:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453'
|
|
108
|
-
end
|
|
103
|
+
context '"Combined" access log parsing' do
|
|
109
104
|
|
|
110
|
-
|
|
111
|
-
@file_format
|
|
105
|
+
before(:all) do
|
|
106
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
|
|
107
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
|
|
108
|
+
@sample_1 = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
|
|
109
|
+
@sample_2 = '10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"'
|
|
110
|
+
@nonsense_sample = 'addasdsasadadssadasd'
|
|
112
111
|
end
|
|
113
112
|
|
|
114
113
|
it "should have a valid language definitions" do
|
|
115
114
|
@file_format.should be_valid
|
|
116
115
|
end
|
|
117
116
|
|
|
118
|
-
it "should parse a valid access log line" do
|
|
119
|
-
@file_format.line_definitions[:access].matches(@sample_1).should be_kind_of(Hash)
|
|
120
|
-
end
|
|
121
|
-
|
|
122
117
|
it "should not parse a valid access log line" do
|
|
123
|
-
@file_format.
|
|
118
|
+
@file_format.should_not parse_line(@nonsense_sample)
|
|
124
119
|
end
|
|
125
120
|
|
|
126
121
|
it "should read the correct values from a valid 404 access log line" do
|
|
127
|
-
@
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
122
|
+
@file_format.should parse_line(@sample_1).as(:access).and_capture(
|
|
123
|
+
:remote_host => '69.41.0.45',
|
|
124
|
+
:remote_logname => nil,
|
|
125
|
+
:user => nil,
|
|
126
|
+
:timestamp => 20090902120240,
|
|
127
|
+
:http_status => 404,
|
|
128
|
+
:http_method => 'GET',
|
|
129
|
+
:http_version => '1.1',
|
|
130
|
+
:bytes_sent => 209,
|
|
131
|
+
:referer => nil,
|
|
132
|
+
:user_agent => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
it "should read the correct values from a valid 200 access log line" do
|
|
136
|
+
@file_format.should parse_line(@sample_2).as(:access).and_capture(
|
|
137
|
+
:remote_host => '10.0.1.1',
|
|
138
|
+
:remote_logname => nil,
|
|
139
|
+
:user => nil,
|
|
140
|
+
:timestamp => 20090902050833,
|
|
141
|
+
:http_status => 200,
|
|
142
|
+
:http_method => 'GET',
|
|
143
|
+
:http_version => '1.1',
|
|
144
|
+
:bytes_sent => 30,
|
|
145
|
+
:referer => nil,
|
|
146
|
+
:user_agent => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
it "should parse 5 request from fixture access log without warnings" do
|
|
150
|
+
request_counter.should_receive(:hit!).exactly(5).times
|
|
151
|
+
@log_parser.should_not_receive(:warn)
|
|
152
|
+
|
|
153
|
+
@log_parser.parse_file(log_fixture(:apache_combined)) do |request|
|
|
154
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
|
|
136
155
|
end
|
|
137
156
|
end
|
|
138
157
|
end
|
|
139
|
-
|
|
140
|
-
context '"
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
@file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
|
|
158
|
+
|
|
159
|
+
context '"Rack" access log parser' do
|
|
160
|
+
before(:each) do
|
|
161
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:rack)
|
|
144
162
|
@log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
|
|
145
|
-
@sample_1 = '
|
|
146
|
-
@sample_2 = '
|
|
163
|
+
@sample_1 = '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453'
|
|
164
|
+
@sample_2 = '127.0.0.1 - - [01/Oct/2009 07:58:10] "GET / HTTP/1.1" 200 1 0.0045'
|
|
165
|
+
@nonsense_sample = 'addasdsasadadssadasd'
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
it "should create a kind of an Apache file format" do
|
|
169
|
+
@file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
|
|
147
170
|
end
|
|
148
171
|
|
|
149
172
|
it "should have a valid language definitions" do
|
|
150
173
|
@file_format.should be_valid
|
|
151
174
|
end
|
|
152
175
|
|
|
153
|
-
it "should parse a valid access log line" do
|
|
154
|
-
@file_format.
|
|
176
|
+
it "should parse a valid access log line with status 500" do
|
|
177
|
+
@file_format.should parse_line(@sample_1).as(:access).and_capture(
|
|
178
|
+
:remote_host => '127.0.0.1', :timestamp => 20090916074008, :user => nil,
|
|
179
|
+
:http_status => 500, :http_method => 'GET', :http_version => '1.1',
|
|
180
|
+
:duration => 0.0453, :bytes_sent => 63183, :remote_logname => nil)
|
|
155
181
|
end
|
|
156
182
|
|
|
157
|
-
it "should
|
|
158
|
-
@file_format.
|
|
183
|
+
it "should parse a valid access log line with status 200" do
|
|
184
|
+
@file_format.should parse_line(@sample_2).as(:access).and_capture(
|
|
185
|
+
:remote_host => '127.0.0.1', :timestamp => 20091001075810, :user => nil,
|
|
186
|
+
:http_status => 200, :http_method => 'GET', :http_version => '1.1',
|
|
187
|
+
:duration => 0.0045, :bytes_sent => 1, :remote_logname => nil)
|
|
159
188
|
end
|
|
160
189
|
|
|
161
|
-
it "should
|
|
162
|
-
@
|
|
163
|
-
request[:remote_host].should == '69.41.0.45'
|
|
164
|
-
request[:timestamp].should == 20090902120240
|
|
165
|
-
request[:http_status].should == 404
|
|
166
|
-
request[:http_method].should == 'GET'
|
|
167
|
-
request[:http_version].should == '1.1'
|
|
168
|
-
request[:bytes_sent].should == 209
|
|
169
|
-
request[:referer].should == nil
|
|
170
|
-
request[:user].should == nil
|
|
171
|
-
request[:user_agent].should == 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)'
|
|
172
|
-
end
|
|
190
|
+
it "should not parse an invalid access log line" do
|
|
191
|
+
@file_format.should_not parse_line(@nonsense_sample)
|
|
173
192
|
end
|
|
174
193
|
|
|
175
|
-
it "should
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
request
|
|
181
|
-
request[:http_version].should == '1.1'
|
|
182
|
-
request[:bytes_sent].should == 30
|
|
183
|
-
request[:referer].should == nil
|
|
184
|
-
request[:user].should == nil
|
|
185
|
-
request[:user_agent].should == 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9'
|
|
194
|
+
it "should parse 2 Apache requests from a sample without warnings" do
|
|
195
|
+
request_counter.should_receive(:hit!).twice
|
|
196
|
+
@log_parser.should_not_receive(:warn)
|
|
197
|
+
|
|
198
|
+
@log_parser.parse_io(log_stream(@sample_1, @nonsense_sample, @sample_2)) do |request|
|
|
199
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
|
|
186
200
|
end
|
|
187
201
|
end
|
|
188
|
-
|
|
189
|
-
it "should parse 5 request from fixture access log" do
|
|
190
|
-
counter = mock('counter')
|
|
191
|
-
counter.should_receive(:hit!).exactly(5).times
|
|
192
|
-
@log_parser.parse_file(log_fixture(:apache_combined)) { counter.hit! }
|
|
193
|
-
end
|
|
194
202
|
end
|
|
195
203
|
end
|
|
196
|
-
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
require File.dirname(__FILE__) + '/../../spec_helper'
|
|
2
|
+
|
|
3
|
+
describe RequestLogAnalyzer::FileFormat do
|
|
4
|
+
|
|
5
|
+
describe '.autodetect' do
|
|
6
|
+
it "should autodetect a Merb log" do
|
|
7
|
+
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:merb))
|
|
8
|
+
file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Merb)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
it "should autodetect a MySQL slow query log" do
|
|
12
|
+
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:mysql_slow_query))
|
|
13
|
+
file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
it "should autodetect a Rails 1.x log" do
|
|
17
|
+
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_1x))
|
|
18
|
+
file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
it "should autodetect a Rails 2.x log" do
|
|
22
|
+
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_22))
|
|
23
|
+
file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
it "should autodetect an Apache access log" do
|
|
27
|
+
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:apache_common))
|
|
28
|
+
file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
it "should not find any file format with a bogus file" do
|
|
32
|
+
RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:test_order)).should be_nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
require File.dirname(__FILE__) + '/../../spec_helper'
|
|
2
|
+
|
|
3
|
+
describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
4
|
+
|
|
5
|
+
it "should be a valid file format" do
|
|
6
|
+
RequestLogAnalyzer::FileFormat.load(:mysql).should be_valid
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
describe '#parse_line' do
|
|
10
|
+
before(:each) do
|
|
11
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:mysql)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it "should parse a :time line correctly" do
|
|
15
|
+
line = '# Time: 091112 8:13:56'
|
|
16
|
+
line = '# Time: 091112 8:13:56'
|
|
17
|
+
@file_format.should parse_line(line).as(:time).and_capture(:timestamp => 20091112081356)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it "should parse a :user_host line correctly with IP present" do
|
|
21
|
+
line = '# User@Host: admin[admin] @ db1 [10.0.0.1]'
|
|
22
|
+
@file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => 'db1', :ip => '10.0.0.1')
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
it "should parse a :user_host line correctly without a host" do
|
|
26
|
+
line = '# User@Host: admin[admin] @ [10.0.0.1]'
|
|
27
|
+
@file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => '', :ip => '10.0.0.1')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
it "should parse a :user_host line correctly with IP absent" do
|
|
31
|
+
line = '# User@Host: root[root] @ localhost []'
|
|
32
|
+
@file_format.should parse_line(line).as(:user_host).and_capture(:user => "root", :host => 'localhost', :ip => "")
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it "should parse a :query_statistics line" do
|
|
36
|
+
line = '# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307'
|
|
37
|
+
@file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
|
|
38
|
+
:lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
it "should parse a :query_statistics line with floating point durations" do
|
|
42
|
+
line = '# Query_time: 10.00000 Lock_time: 0.00000 Rows_sent: 1191307 Rows_examined: 1191307'
|
|
43
|
+
@file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
|
|
44
|
+
:lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
it "should parse a :query_part line" do
|
|
48
|
+
line = ' AND clients.index > 0'
|
|
49
|
+
@file_format.should parse_line(line).as(:query_part).and_capture(:query_fragment => line)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
it "should parse a final :query line" do
|
|
53
|
+
line = 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`; '
|
|
54
|
+
@file_format.should parse_line(line).as(:query).and_capture(:query =>
|
|
55
|
+
'SELECT /*!:int SQL_NO_CACHE */ * FROM events')
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it "should parse a :use_database line" do
|
|
59
|
+
line = 'use db;'
|
|
60
|
+
@file_format.should parse_line(line).as(:use_database).and_capture(:database => 'db')
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
it "should not parse a SET timestamp line" do
|
|
64
|
+
line = 'SET timestamp=1250651725;'
|
|
65
|
+
@file_format.should_not parse_line(line)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
it "should not parse a SET insert_id line" do
|
|
69
|
+
line = 'SET insert_id=1250651725;'
|
|
70
|
+
@file_format.should_not parse_line(line)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it "should not parse a SET timestamp, insert_id line" do
|
|
74
|
+
line = 'SET timestamp=1250651725, insert_id=45674;'
|
|
75
|
+
@file_format.should_not parse_line(line)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
describe '#parse_io' do
|
|
82
|
+
before(:each) do
|
|
83
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:mysql))
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
it "should parse a single line query entry correctly" do
|
|
87
|
+
fixture = <<EOS
|
|
88
|
+
# Time: 091112 18:13:56
|
|
89
|
+
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
90
|
+
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
91
|
+
SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
|
|
92
|
+
EOS
|
|
93
|
+
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
94
|
+
request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
|
|
95
|
+
request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
it "should parse a multiline query entry correctly" do
|
|
100
|
+
fixture = <<EOS
|
|
101
|
+
# Time: 091112 18:13:56
|
|
102
|
+
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
103
|
+
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
104
|
+
SELECT * FROM `clients` WHERE (1=1
|
|
105
|
+
AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
|
|
106
|
+
AND clients.index > 0
|
|
107
|
+
) AND (clients.deleted_at IS NULL);
|
|
108
|
+
EOS
|
|
109
|
+
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
110
|
+
request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
|
|
111
|
+
request[:query].should == "SELECT * FROM clients WHERE (:int=:int AND clients.valid_from < :date AND (clients.valid_to IS NULL or clients.valid_to > :date) AND clients.index > :int ) AND (clients.deleted_at IS NULL)"
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
it "should parse a request without timestamp correctly" do
|
|
116
|
+
fixture = <<EOS
|
|
117
|
+
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
118
|
+
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
119
|
+
SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
|
|
120
|
+
EOS
|
|
121
|
+
request_counter.should_receive(:hit!).once
|
|
122
|
+
@log_parser.should_not_receive(:warn)
|
|
123
|
+
|
|
124
|
+
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
125
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
it "should parse a query with context information correctly" do
|
|
130
|
+
fixture = <<EOS
|
|
131
|
+
# Time: 091112 18:13:56
|
|
132
|
+
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
133
|
+
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
134
|
+
use database_name;
|
|
135
|
+
SET timestamp=4324342342423, insert_id = 224253443;
|
|
136
|
+
SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
|
|
137
|
+
EOS
|
|
138
|
+
|
|
139
|
+
request_counter.should_receive(:hit!).once
|
|
140
|
+
@log_parser.should_not_receive(:warn)
|
|
141
|
+
|
|
142
|
+
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
143
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
|
|
144
|
+
request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
it "should find 26 completed sloq query entries" do
|
|
151
|
+
@log_parser.should_receive(:handle_request).exactly(26).times
|
|
152
|
+
@log_parser.parse_file(log_fixture(:mysql_slow_query))
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|