request-log-analyzer 1.7.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -1
- data/DESIGN.rdoc +0 -1
- data/Gemfile +2 -0
- data/README.rdoc +16 -5
- data/bin/request-log-analyzer +4 -2
- data/lib/cli/database_console_init.rb +1 -1
- data/lib/cli/tools.rb +2 -1
- data/lib/mixins/gets_memory_protection.rb +1 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -2
- data/lib/request_log_analyzer/aggregator.rb +3 -3
- data/lib/request_log_analyzer/controller.rb +42 -36
- data/lib/request_log_analyzer/database.rb +5 -3
- data/lib/request_log_analyzer/file_format/apache.rb +19 -28
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +38 -0
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +39 -0
- data/lib/request_log_analyzer/file_format/haproxy.rb +138 -0
- data/lib/request_log_analyzer/file_format/oink.rb +96 -0
- data/lib/request_log_analyzer/file_format/postgresql.rb +7 -6
- data/lib/request_log_analyzer/file_format/rails.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails3.rb +34 -11
- data/lib/request_log_analyzer/file_format/w3c.rb +56 -0
- data/lib/request_log_analyzer/file_format.rb +33 -10
- data/lib/request_log_analyzer/filter.rb +3 -5
- data/lib/request_log_analyzer/output/fancy_html.rb +0 -5
- data/lib/request_log_analyzer/output.rb +6 -7
- data/lib/request_log_analyzer/request.rb +17 -3
- data/lib/request_log_analyzer/source/log_parser.rb +3 -1
- data/lib/request_log_analyzer/source.rb +2 -6
- data/lib/request_log_analyzer/tracker/duration.rb +7 -0
- data/lib/request_log_analyzer/tracker/numeric_value.rb +123 -12
- data/lib/request_log_analyzer/tracker/traffic.rb +7 -0
- data/lib/request_log_analyzer/tracker.rb +6 -5
- data/lib/request_log_analyzer.rb +36 -17
- data/request-log-analyzer.gemspec +9 -6
- data/spec/fixtures/oink_22.log +19 -0
- data/spec/fixtures/oink_22_failure.log +17 -0
- data/spec/fixtures/postgresql.log +29 -2980
- data/spec/integration/command_line_usage_spec.rb +4 -4
- data/spec/integration/mailer_spec.rb +20 -4
- data/spec/integration/munin_plugins_rails_spec.rb +1 -1
- data/spec/integration/scout_spec.rb +1 -1
- data/spec/lib/helpers.rb +2 -2
- data/spec/lib/macros.rb +1 -1
- data/spec/lib/matchers.rb +1 -1
- data/spec/lib/mocks.rb +1 -1
- data/spec/spec_helper.rb +10 -10
- data/spec/unit/aggregator/database_inserter_spec.rb +1 -1
- data/spec/unit/aggregator/summarizer_spec.rb +1 -1
- data/spec/unit/controller/controller_spec.rb +1 -1
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +1 -1
- data/spec/unit/database/connection_spec.rb +1 -1
- data/spec/unit/database/database_spec.rb +1 -1
- data/spec/unit/file_format/amazon_s3_format_spec.rb +1 -1
- data/spec/unit/file_format/apache_format_spec.rb +18 -1
- data/spec/unit/file_format/common_regular_expressions_spec.rb +1 -1
- data/spec/unit/file_format/delayed_job21_format_spec.rb +60 -0
- data/spec/unit/file_format/delayed_job2_format_spec.rb +60 -0
- data/spec/unit/file_format/delayed_job_format_spec.rb +2 -2
- data/spec/unit/file_format/file_format_api_spec.rb +2 -2
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +84 -0
- data/spec/unit/file_format/line_definition_spec.rb +1 -1
- data/spec/unit/file_format/merb_format_spec.rb +1 -1
- data/spec/unit/file_format/mysql_format_spec.rb +1 -1
- data/spec/unit/file_format/oink_format_spec.rb +105 -0
- data/spec/unit/file_format/postgresql_format_spec.rb +24 -46
- data/spec/unit/file_format/rack_format_spec.rb +1 -1
- data/spec/unit/file_format/rails3_format_spec.rb +41 -14
- data/spec/unit/file_format/rails_format_spec.rb +11 -1
- data/spec/unit/file_format/w3c_format_spec.rb +46 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +1 -1
- data/spec/unit/filter/field_filter_spec.rb +1 -1
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +1 -1
- data/spec/unit/mailer_spec.rb +1 -1
- data/spec/unit/request_spec.rb +1 -1
- data/spec/unit/source/log_parser_spec.rb +1 -1
- data/spec/unit/tracker/duration_tracker_spec.rb +4 -2
- data/spec/unit/tracker/frequency_tracker_spec.rb +1 -1
- data/spec/unit/tracker/hourly_spread_spec.rb +1 -1
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +38 -17
- data/spec/unit/tracker/timespan_tracker_spec.rb +1 -1
- data/spec/unit/tracker/tracker_api_spec.rb +1 -1
- data/spec/unit/tracker/traffic_tracker_spec.rb +1 -1
- data/tasks/github-gem.rake +97 -60
- data/tasks/request_log_analyzer.rake +102 -0
- metadata +198 -153
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe RequestLogAnalyzer::FileFormat::Oink do
|
|
4
|
+
describe '.create' do
|
|
5
|
+
|
|
6
|
+
context 'without providing a lines argument' do
|
|
7
|
+
before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink) }
|
|
8
|
+
|
|
9
|
+
it "should create a valid file format" do
|
|
10
|
+
@oink.should be_valid
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it "should parse :memory_usage line" do
|
|
14
|
+
@oink.line_definitions.should include(:memory_usage)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
describe '#parse_line' do
|
|
22
|
+
before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink, :all) }
|
|
23
|
+
|
|
24
|
+
it "should parse a :memory_usage line correctly" do
|
|
25
|
+
line = 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783'
|
|
26
|
+
@oink.should parse_line(line).as(:memory_usage).and_capture(:pid => 67783, :memory => 714052)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
it "should parse the PID from a :processing line correctly" do
|
|
30
|
+
line = 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
|
|
31
|
+
@oink.should parse_line(line).as(:processing).and_capture(:pid => 67783, :controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET', :ip => '1.1.1.1')
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
it "should parse a :instance_type_counter correctly" do
|
|
35
|
+
|
|
36
|
+
line = "Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1"
|
|
37
|
+
|
|
38
|
+
@oink.should parse_line(line).as(:instance_type_counter).and_capture(:pid => 26364, :instance_counts => {'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1})
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
describe '#parse_io' do
|
|
43
|
+
context "Rails 2.2 style log" do
|
|
44
|
+
before(:each) do
|
|
45
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(
|
|
46
|
+
RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it "should parse requests" do
|
|
50
|
+
request_counter.should_receive(:hit!).exactly(4).times
|
|
51
|
+
|
|
52
|
+
@log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
53
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
it "should not record :memory_diff on first request" do
|
|
58
|
+
@log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
59
|
+
if @log_parser.parsed_requests == 1
|
|
60
|
+
request[:memory_diff].should == nil
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
it "should record :memory_diff of 2nd tracked PID" do
|
|
66
|
+
@log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
67
|
+
if @log_parser.parsed_requests == 3
|
|
68
|
+
request[:memory_diff].should == 50000*1024
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
it "should record :memory_diff of 1st tracked PID" do
|
|
74
|
+
@log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
75
|
+
if @log_parser.parsed_requests == 4
|
|
76
|
+
request[:memory_diff].should == 30000*1024
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
context 'Rails 2.2 style log w/failure' do
|
|
83
|
+
before(:each) do
|
|
84
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(
|
|
85
|
+
RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
it "should parse requests" do
|
|
89
|
+
request_counter.should_receive(:hit!).exactly(4).times
|
|
90
|
+
|
|
91
|
+
@log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
|
|
92
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
it "should ignore memory changes when a failure occurs" do
|
|
97
|
+
@log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
|
|
98
|
+
if @log_parser.parsed_requests == 4
|
|
99
|
+
request[:memory_diff].should == nil
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
@@ -1,65 +1,43 @@
|
|
|
1
|
-
require
|
|
1
|
+
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Postgresql do
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
|
|
6
|
+
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
7
|
+
|
|
8
|
+
it { should be_valid }
|
|
8
9
|
|
|
9
10
|
describe '#parse_line' do
|
|
10
|
-
before(:each) do
|
|
11
|
-
@file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
|
|
12
|
-
end
|
|
13
|
-
|
|
14
11
|
it "should parse a :query line correctly" do
|
|
15
|
-
line = '
|
|
16
|
-
|
|
12
|
+
line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
|
|
13
|
+
subject.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
|
|
17
14
|
end
|
|
18
15
|
|
|
19
16
|
it "should parse a :query_fragment line correctly" do
|
|
20
|
-
line = '
|
|
21
|
-
|
|
17
|
+
line = ' ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
|
|
18
|
+
subject.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
|
|
22
19
|
end
|
|
23
20
|
|
|
24
|
-
it "should parse a :
|
|
25
|
-
line = '
|
|
26
|
-
|
|
27
|
-
end
|
|
21
|
+
it "should parse a :query line correctly" do
|
|
22
|
+
line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
|
|
23
|
+
subject.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
|
|
24
|
+
end
|
|
28
25
|
end
|
|
29
26
|
|
|
30
27
|
describe '#parse_io' do
|
|
31
|
-
before(:each) do
|
|
32
|
-
@log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
|
|
33
|
-
end
|
|
34
|
-
|
|
35
28
|
it "should parse a multiline query entry correctly" do
|
|
36
|
-
fixture =
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
48
|
-
request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
|
|
49
|
-
request[:query].should == "SELECT groups.group_name,groups.unix_group_name, groups.type_id,users.user_name,users.realname, news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details FROM users,news_bytes,groups WHERE news_bytes.group_id=:string AND news_bytes.is_approved <> :string AND users.user_id=news_bytes.submitted_by AND news_bytes.group_id=groups.group_id ORDER BY post_date DESC LIMIT :int OFFSET :int"
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
it "should parse a dualline query entry correctly" do
|
|
54
|
-
fixture = <<EOS
|
|
55
|
-
2004-05-07 11:58:36 LOG: query: SELECT type, count FROM project_sums_agg WHERE group_id=59
|
|
56
|
-
2004-05-07 11:58:36 LOG: duration: 0.001197 sec
|
|
57
|
-
EOS
|
|
58
|
-
@log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
59
|
-
request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
|
|
60
|
-
request[:query].should == "SELECT type, count FROM project_sums_agg WHERE group_id=:int"
|
|
29
|
+
fixture = <<-EOS
|
|
30
|
+
2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG: 00000: duration: 0.195 ms statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {}
|
|
31
|
+
', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id"
|
|
32
|
+
2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081
|
|
33
|
+
EOS
|
|
34
|
+
|
|
35
|
+
request_counter.should_receive(:hit!).exactly(1).times
|
|
36
|
+
log_parser.should_not_receive(:warn)
|
|
37
|
+
log_parser.parse_io(StringIO.new(fixture)) do |request|
|
|
38
|
+
request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Postgresql::Request) && request.completed?
|
|
39
|
+
request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
|
|
61
40
|
end
|
|
62
41
|
end
|
|
63
42
|
end
|
|
64
43
|
end
|
|
65
|
-
|
|
@@ -10,30 +10,57 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
10
10
|
before(:each) { @file_format = RequestLogAnalyzer::FileFormat.load(:rails3) }
|
|
11
11
|
|
|
12
12
|
it "should parse :started lines correctly" do
|
|
13
|
-
line = 'Started GET "/queries" for 127.0.0.1 at
|
|
13
|
+
line = 'Started GET "/queries" for 127.0.0.1 at Thu Feb 25 16:15:18 -0800 2010'
|
|
14
14
|
@file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
|
|
15
15
|
:path => '/queries', :ip => '127.0.0.1', :timestamp => 20100225161518)
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
it "should parse :started lines in Oct, Nov and Dec correctly" do
|
|
19
|
+
line = 'Started GET "/queries" for 127.0.0.1 at Thu Oct 25 16:15:18 -0800 2010'
|
|
20
|
+
@file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
|
|
21
|
+
:path => '/queries', :ip => '127.0.0.1', :timestamp => 20101025161518)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it "should parse :started lines in Ruby 1.9.2 format correctly" do
|
|
25
|
+
line = 'Started GET "/queries" for 127.0.0.1 at 2010-10-26 02:27:15 +0000'
|
|
26
|
+
@file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
|
|
27
|
+
:path => '/queries', :ip => '127.0.0.1', :timestamp => 20101026022715)
|
|
28
|
+
end
|
|
29
|
+
|
|
18
30
|
it "should parse :processing lines correctly" do
|
|
19
31
|
line = ' Processing by QueriesController#index as HTML'
|
|
20
32
|
@file_format.should parse_line(line).as(:processing).and_capture(
|
|
21
33
|
:controller => 'QueriesController', :action => 'index', :format => 'HTML')
|
|
22
34
|
end
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
35
|
+
it "should parse nested :processing lines correctly" do
|
|
36
|
+
line = ' Processing by Projects::QueriesController#index as HTML'
|
|
37
|
+
@file_format.should parse_line(line).as(:processing).and_capture(
|
|
38
|
+
:controller => 'Projects::QueriesController', :action => 'index', :format => 'HTML')
|
|
39
|
+
end
|
|
40
|
+
it "should parse :processing lines correctly with format */*" do
|
|
41
|
+
line = ' Processing by ProjectsController#avatar as */*'
|
|
42
|
+
@file_format.should parse_line(line).as(:processing).and_capture(
|
|
43
|
+
:controller => 'ProjectsController', :action => 'avatar', :format => '*/*')
|
|
44
|
+
end
|
|
30
45
|
|
|
31
46
|
it "should parse :completed lines correctly" do
|
|
32
|
-
line = 'Completed 200 OK in 170ms (Views: 78.
|
|
47
|
+
line = 'Completed 200 OK in 170ms (Views: 78.0ms | ActiveRecord: 48.2ms)'
|
|
33
48
|
@file_format.should parse_line(line).as(:completed).and_capture(
|
|
34
|
-
:duration => 0.170, :status => 200)
|
|
49
|
+
:duration => 0.170, :view => 0.078, :db => 0.0482, :status => 200)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
it "should parse :completed lines correctly when ActiveRecord is not mentioned" do
|
|
53
|
+
line = 'Completed 200 OK in 364ms (Views: 31.4ms)'
|
|
54
|
+
@file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.364, :status => 200)
|
|
35
55
|
end
|
|
36
56
|
|
|
57
|
+
it "should parse :completed lines correctly when other durations are specified" do
|
|
58
|
+
line = 'Completed 200 OK in 384ms (Views: 222.0ms | ActiveRecord: 121.0ms | Sphinx: 0.0ms)'
|
|
59
|
+
@file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.384, :view => 0.222,
|
|
60
|
+
:db => 0.121, :status => 200)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
|
|
37
64
|
it "should pase :failure lines correctly" do
|
|
38
65
|
line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"
|
|
39
66
|
@file_format.should parse_line(line).as(:failure).and_capture(:line => 3,
|
|
@@ -50,7 +77,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
50
77
|
|
|
51
78
|
it "should parse a successful request correctly" do
|
|
52
79
|
log = <<-EOLOG
|
|
53
|
-
Started GET "/" for 127.0.0.1 at
|
|
80
|
+
Started GET "/" for 127.0.0.1 at Fri Mar 19 06:40:41 -0700 2010
|
|
54
81
|
Processing by QueriesController#index as HTML
|
|
55
82
|
SQL (16.3ms) SHOW TABLES
|
|
56
83
|
Query Load (32.0ms) SELECT `queries`.* FROM `queries`
|
|
@@ -68,7 +95,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
68
95
|
|
|
69
96
|
it "should parse an unroutable request correctly" do
|
|
70
97
|
log = <<-EOLOG
|
|
71
|
-
Started GET "/404" for 127.0.0.1 at
|
|
98
|
+
Started GET "/404" for 127.0.0.1 at Fri Mar 19 06:40:57 -0700 2010
|
|
72
99
|
|
|
73
100
|
ActionController::RoutingError (No route matches "/404"):
|
|
74
101
|
|
|
@@ -87,7 +114,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
87
114
|
|
|
88
115
|
it "should parse a failing request correctly" do
|
|
89
116
|
log = <<-EOLOG
|
|
90
|
-
Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at
|
|
117
|
+
Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at Mon Mar 01 18:44:33 -0800 2010
|
|
91
118
|
Processing by QueriesController#execute as CSV
|
|
92
119
|
Parameters: {"commit"=>"Run query", "authenticity_token"=>"pz9WcxkcrlG/43eg6BgSAnJL7yIsaffuHbYxPHUsUzQ=", "id"=>"397638749"}
|
|
93
120
|
|
|
@@ -117,4 +144,4 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
117
144
|
end
|
|
118
145
|
end
|
|
119
146
|
end
|
|
120
|
-
end
|
|
147
|
+
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require
|
|
1
|
+
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Rails do
|
|
4
4
|
|
|
@@ -75,6 +75,16 @@ describe RequestLogAnalyzer::FileFormat::Rails do
|
|
|
75
75
|
@rails.should parse_line(line).as(:completed).and_capture(:duration => 0.614, :db => 0.031, :view => 0.120, :status => 200, :url => 'http://floorplanner.local/demo')
|
|
76
76
|
end
|
|
77
77
|
|
|
78
|
+
it "should parse a Rails 2.2 style :completed line correctly when AR is disabled" do
|
|
79
|
+
line = prefix + 'Completed in 597ms (View: 298 | 200 OK [http://shapado.com]'
|
|
80
|
+
@rails.should parse_line(line).as(:completed).and_capture(:duration => 0.597, :db => nil, :view => 0.298, :status => 200, :url => 'http://shapado.com')
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
it "should parse a Rails 2.2 style :completed line without view" do
|
|
84
|
+
line = prefix + "Completed in 148ms (DB: 0) | 302 Found [http://iwp-sod.hargray.org/login]"
|
|
85
|
+
@rails.should parse_line(line).as(:completed).and_capture(:duration => 0.148, :db => 0.0, :view => nil, :status => 302, :url => 'http://iwp-sod.hargray.org/login')
|
|
86
|
+
end
|
|
87
|
+
|
|
78
88
|
it "should parse a :failure line with exception correctly" do
|
|
79
89
|
line = prefix + "NoMethodError (undefined method `update_domain_account' for nil:NilClass):"
|
|
80
90
|
@rails.should parse_line(line).as(:failure).and_capture(:error => 'NoMethodError', :message => "undefined method `update_domain_account' for nil:NilClass")
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
describe RequestLogAnalyzer::FileFormat::W3c do
|
|
4
|
+
|
|
5
|
+
before(:each) do
|
|
6
|
+
@file_format = RequestLogAnalyzer::FileFormat.load(:w3c)
|
|
7
|
+
@log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
|
|
8
|
+
# date time c-ip cs-username s-ip s-port cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(User-Agent) cs(Referrer)
|
|
9
|
+
@sample = '2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/'
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
it "should be a valid file format" do
|
|
13
|
+
@file_format.should be_valid
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
it "should parse access lines and capture all of its fields" do
|
|
17
|
+
@file_format.should have_line_definition(:access).capturing(:timestamp, :remote_ip, :username, :local_ip, :port, :method, :path, :http_status, :bytes_sent, :bytes_received, :duration, :user_agent, :referer)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it "should match the sample line" do
|
|
21
|
+
@file_format.parse_line(@sample).should include(:line_definition, :captures)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it "should not match a nonsense line" do
|
|
25
|
+
@file_format.parse_line('#Software: Microsoft Internet Information Services 6.0').should be_nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it "should parse and convert the sample fields correctly" do
|
|
29
|
+
@log_parser.parse_io(StringIO.new(@sample)) do |request|
|
|
30
|
+
request[:timestamp].should == 20020524201801
|
|
31
|
+
request[:remote_ip].should == "172.224.24.114"
|
|
32
|
+
request[:username].should == nil
|
|
33
|
+
request[:local_ip].should == "206.73.118.24"
|
|
34
|
+
request[:port].should == 80
|
|
35
|
+
request[:method].should == 'GET'
|
|
36
|
+
request[:path].should == '/Default.htm'
|
|
37
|
+
request[:http_status].should == 200
|
|
38
|
+
request[:bytes_sent].should == 7930
|
|
39
|
+
request[:bytes_received].should == 248
|
|
40
|
+
request[:duration].should == 0.031
|
|
41
|
+
request[:user_agent].should == 'Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server)'
|
|
42
|
+
request[:referer].should == 'http://64.224.24.114/'
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
end
|
data/spec/unit/mailer_spec.rb
CHANGED
data/spec/unit/request_spec.rb
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require
|
|
1
|
+
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::Tracker::Duration do
|
|
4
4
|
|
|
@@ -27,7 +27,9 @@ describe RequestLogAnalyzer::Tracker::Duration do
|
|
|
27
27
|
it "should generate a YAML output" do
|
|
28
28
|
@tracker.update(request(:category => 'a', :duration => 0.2))
|
|
29
29
|
@tracker.update(request(:category => 'b', :duration => 0.2))
|
|
30
|
-
@tracker.to_yaml_object.should
|
|
30
|
+
@tracker.to_yaml_object.keys.should =~ ['a', 'b']
|
|
31
|
+
@tracker.to_yaml_object['a'].should include(:min => 0.2, :hits => 1, :max => 0.2, :mean => 0.2, :sum => 0.2, :sum_of_squares => 0.0)
|
|
32
|
+
@tracker.to_yaml_object['b'].should include(:min => 0.2, :hits => 1, :max => 0.2, :mean => 0.2, :sum => 0.2, :sum_of_squares => 0.0)
|
|
31
33
|
end
|
|
32
34
|
end
|
|
33
35
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require
|
|
1
|
+
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
4
4
|
|
|
@@ -22,7 +22,7 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
22
22
|
@tracker.categories['b'][:sum].should == 4
|
|
23
23
|
end
|
|
24
24
|
end
|
|
25
|
-
|
|
25
|
+
|
|
26
26
|
|
|
27
27
|
context 'dynamic category' do
|
|
28
28
|
before(:each) do
|
|
@@ -40,11 +40,12 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
40
40
|
@tracker.categories['slow'][:sum].should == 4
|
|
41
41
|
end
|
|
42
42
|
end
|
|
43
|
-
|
|
44
|
-
describe '#update' do
|
|
45
43
|
|
|
44
|
+
describe '#update' do
|
|
46
45
|
before(:each) do
|
|
47
|
-
@tracker = RequestLogAnalyzer::Tracker::NumericValue.new(:value => :duration, :category => :category
|
|
46
|
+
@tracker = RequestLogAnalyzer::Tracker::NumericValue.new(:value => :duration, :category => :category,
|
|
47
|
+
:min_bucket_value => 0.0001, :max_bucket_value => 1000)
|
|
48
|
+
|
|
48
49
|
@tracker.prepare
|
|
49
50
|
|
|
50
51
|
@tracker.update(request(:category => 'a', :duration => 0.4))
|
|
@@ -53,11 +54,11 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
53
54
|
end
|
|
54
55
|
|
|
55
56
|
it "should sum of the durations for a category correctly" do
|
|
56
|
-
@tracker.sum('a').should
|
|
57
|
+
@tracker.sum('a').should be_within(0.000001).of(0.9)
|
|
57
58
|
end
|
|
58
59
|
|
|
59
60
|
it "should overall sum of the durations correctly" do
|
|
60
|
-
@tracker.sum_overall.should
|
|
61
|
+
@tracker.sum_overall.should be_within(0.000001).of(0.9)
|
|
61
62
|
end
|
|
62
63
|
|
|
63
64
|
it "should keep track of the minimum and maximum duration" do
|
|
@@ -66,21 +67,39 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
66
67
|
end
|
|
67
68
|
|
|
68
69
|
it "should calculate the mean duration correctly" do
|
|
69
|
-
@tracker.mean('a').should
|
|
70
|
+
@tracker.mean('a').should be_within(0.000001).of(0.3)
|
|
70
71
|
end
|
|
71
72
|
|
|
72
73
|
it "should calculate the overall mean duration correctly" do
|
|
73
|
-
@tracker.mean_overall.should
|
|
74
|
+
@tracker.mean_overall.should be_within(0.000001).of(0.3)
|
|
74
75
|
end
|
|
75
76
|
|
|
76
77
|
it "should calculate the duration variance correctly" do
|
|
77
|
-
@tracker.variance('a').should
|
|
78
|
+
@tracker.variance('a').should be_within(0.000001).of(0.01)
|
|
78
79
|
end
|
|
79
80
|
|
|
80
81
|
it "should calculate the duration standard deviation correctly" do
|
|
81
|
-
@tracker.stddev('a').should
|
|
82
|
+
@tracker.stddev('a').should be_within(0.000001).of(0.1)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
it "should calculate the bucket spread correctly" do
|
|
86
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
87
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
88
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
89
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
90
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
91
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
92
|
+
@tracker.update(request(:category => 'a', :duration => 0.3))
|
|
93
|
+
# 0.2, 0.3 and 0.4 are already there, so, 10 values in total
|
|
94
|
+
|
|
95
|
+
@tracker.median('a').should be_within(0.01).of(0.3)
|
|
96
|
+
|
|
97
|
+
@tracker.percentile_interval('a', 80).begin.should be_within(0.01).of(0.3)
|
|
98
|
+
@tracker.percentile_interval('a', 80).end.should be_within(0.01).of(0.3)
|
|
99
|
+
@tracker.percentile_interval('a', 90).begin.should be_within(0.01).of(0.2)
|
|
100
|
+
@tracker.percentile_interval('a', 90).end.should be_within(0.01).of(0.4)
|
|
82
101
|
end
|
|
83
|
-
end
|
|
102
|
+
end
|
|
84
103
|
|
|
85
104
|
describe '#report' do
|
|
86
105
|
before(:each) do
|
|
@@ -113,9 +132,13 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
113
132
|
it "should generate a YAML output" do
|
|
114
133
|
@tracker.update(request(:category => 'a', :blah => 2))
|
|
115
134
|
@tracker.update(request(:category => 'b', :blah => 3))
|
|
116
|
-
@tracker.to_yaml_object.should
|
|
117
|
-
|
|
118
|
-
|
|
135
|
+
@tracker.to_yaml_object.keys.should =~ ['a', 'b']
|
|
136
|
+
|
|
137
|
+
@tracker.to_yaml_object['a'].should include(:min => 2, :hits => 1, :max => 2, :mean => 2.0, :sum => 2, :sum_of_squares => 0.0)
|
|
138
|
+
@tracker.to_yaml_object['a'][:interval_95_percent].should be_member(2)
|
|
139
|
+
|
|
140
|
+
@tracker.to_yaml_object['b'].should include(:min => 3, :hits => 1, :max => 3, :mean => 3.0, :sum => 3, :sum_of_squares => 0.0)
|
|
141
|
+
@tracker.to_yaml_object['b'][:interval_95_percent].should be_member(3)
|
|
119
142
|
end
|
|
120
143
|
end
|
|
121
144
|
|
|
@@ -161,6 +184,4 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
|
|
|
161
184
|
@tracker.display_value(9000_000_001_001_000).should eql('9000T')
|
|
162
185
|
end
|
|
163
186
|
end
|
|
164
|
-
|
|
165
|
-
|
|
166
187
|
end
|