request-log-analyzer 1.7.0 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. data/.gitignore +4 -1
  2. data/DESIGN.rdoc +0 -1
  3. data/Gemfile +2 -0
  4. data/README.rdoc +16 -5
  5. data/bin/request-log-analyzer +4 -2
  6. data/lib/cli/database_console_init.rb +1 -1
  7. data/lib/cli/tools.rb +2 -1
  8. data/lib/mixins/gets_memory_protection.rb +1 -1
  9. data/lib/request_log_analyzer/aggregator/summarizer.rb +2 -2
  10. data/lib/request_log_analyzer/aggregator.rb +3 -3
  11. data/lib/request_log_analyzer/controller.rb +42 -36
  12. data/lib/request_log_analyzer/database.rb +5 -3
  13. data/lib/request_log_analyzer/file_format/apache.rb +19 -28
  14. data/lib/request_log_analyzer/file_format/delayed_job2.rb +38 -0
  15. data/lib/request_log_analyzer/file_format/delayed_job21.rb +39 -0
  16. data/lib/request_log_analyzer/file_format/haproxy.rb +138 -0
  17. data/lib/request_log_analyzer/file_format/oink.rb +96 -0
  18. data/lib/request_log_analyzer/file_format/postgresql.rb +7 -6
  19. data/lib/request_log_analyzer/file_format/rails.rb +4 -4
  20. data/lib/request_log_analyzer/file_format/rails3.rb +34 -11
  21. data/lib/request_log_analyzer/file_format/w3c.rb +56 -0
  22. data/lib/request_log_analyzer/file_format.rb +33 -10
  23. data/lib/request_log_analyzer/filter.rb +3 -5
  24. data/lib/request_log_analyzer/output/fancy_html.rb +0 -5
  25. data/lib/request_log_analyzer/output.rb +6 -7
  26. data/lib/request_log_analyzer/request.rb +17 -3
  27. data/lib/request_log_analyzer/source/log_parser.rb +3 -1
  28. data/lib/request_log_analyzer/source.rb +2 -6
  29. data/lib/request_log_analyzer/tracker/duration.rb +7 -0
  30. data/lib/request_log_analyzer/tracker/numeric_value.rb +123 -12
  31. data/lib/request_log_analyzer/tracker/traffic.rb +7 -0
  32. data/lib/request_log_analyzer/tracker.rb +6 -5
  33. data/lib/request_log_analyzer.rb +36 -17
  34. data/request-log-analyzer.gemspec +9 -6
  35. data/spec/fixtures/oink_22.log +19 -0
  36. data/spec/fixtures/oink_22_failure.log +17 -0
  37. data/spec/fixtures/postgresql.log +29 -2980
  38. data/spec/integration/command_line_usage_spec.rb +4 -4
  39. data/spec/integration/mailer_spec.rb +20 -4
  40. data/spec/integration/munin_plugins_rails_spec.rb +1 -1
  41. data/spec/integration/scout_spec.rb +1 -1
  42. data/spec/lib/helpers.rb +2 -2
  43. data/spec/lib/macros.rb +1 -1
  44. data/spec/lib/matchers.rb +1 -1
  45. data/spec/lib/mocks.rb +1 -1
  46. data/spec/spec_helper.rb +10 -10
  47. data/spec/unit/aggregator/database_inserter_spec.rb +1 -1
  48. data/spec/unit/aggregator/summarizer_spec.rb +1 -1
  49. data/spec/unit/controller/controller_spec.rb +1 -1
  50. data/spec/unit/controller/log_processor_spec.rb +1 -1
  51. data/spec/unit/database/base_class_spec.rb +1 -1
  52. data/spec/unit/database/connection_spec.rb +1 -1
  53. data/spec/unit/database/database_spec.rb +1 -1
  54. data/spec/unit/file_format/amazon_s3_format_spec.rb +1 -1
  55. data/spec/unit/file_format/apache_format_spec.rb +18 -1
  56. data/spec/unit/file_format/common_regular_expressions_spec.rb +1 -1
  57. data/spec/unit/file_format/delayed_job21_format_spec.rb +60 -0
  58. data/spec/unit/file_format/delayed_job2_format_spec.rb +60 -0
  59. data/spec/unit/file_format/delayed_job_format_spec.rb +2 -2
  60. data/spec/unit/file_format/file_format_api_spec.rb +2 -2
  61. data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
  62. data/spec/unit/file_format/haproxy_format_spec.rb +84 -0
  63. data/spec/unit/file_format/line_definition_spec.rb +1 -1
  64. data/spec/unit/file_format/merb_format_spec.rb +1 -1
  65. data/spec/unit/file_format/mysql_format_spec.rb +1 -1
  66. data/spec/unit/file_format/oink_format_spec.rb +105 -0
  67. data/spec/unit/file_format/postgresql_format_spec.rb +24 -46
  68. data/spec/unit/file_format/rack_format_spec.rb +1 -1
  69. data/spec/unit/file_format/rails3_format_spec.rb +41 -14
  70. data/spec/unit/file_format/rails_format_spec.rb +11 -1
  71. data/spec/unit/file_format/w3c_format_spec.rb +46 -0
  72. data/spec/unit/filter/anonymize_filter_spec.rb +1 -1
  73. data/spec/unit/filter/field_filter_spec.rb +1 -1
  74. data/spec/unit/filter/filter_spec.rb +1 -1
  75. data/spec/unit/filter/timespan_filter_spec.rb +1 -1
  76. data/spec/unit/mailer_spec.rb +1 -1
  77. data/spec/unit/request_spec.rb +1 -1
  78. data/spec/unit/source/log_parser_spec.rb +1 -1
  79. data/spec/unit/tracker/duration_tracker_spec.rb +4 -2
  80. data/spec/unit/tracker/frequency_tracker_spec.rb +1 -1
  81. data/spec/unit/tracker/hourly_spread_spec.rb +1 -1
  82. data/spec/unit/tracker/numeric_value_tracker_spec.rb +38 -17
  83. data/spec/unit/tracker/timespan_tracker_spec.rb +1 -1
  84. data/spec/unit/tracker/tracker_api_spec.rb +1 -1
  85. data/spec/unit/tracker/traffic_tracker_spec.rb +1 -1
  86. data/tasks/github-gem.rake +97 -60
  87. data/tasks/request_log_analyzer.rake +102 -0
  88. metadata +198 -153
@@ -0,0 +1,105 @@
1
+ require 'spec_helper'
2
+
3
+ describe RequestLogAnalyzer::FileFormat::Oink do
4
+ describe '.create' do
5
+
6
+ context 'without providing a lines argument' do
7
+ before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink) }
8
+
9
+ it "should create a valid file format" do
10
+ @oink.should be_valid
11
+ end
12
+
13
+ it "should parse :memory_usage line" do
14
+ @oink.line_definitions.should include(:memory_usage)
15
+ end
16
+
17
+ end
18
+
19
+ end
20
+
21
+ describe '#parse_line' do
22
+ before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink, :all) }
23
+
24
+ it "should parse a :memory_usage line correctly" do
25
+ line = 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783'
26
+ @oink.should parse_line(line).as(:memory_usage).and_capture(:pid => 67783, :memory => 714052)
27
+ end
28
+
29
+ it "should parse the PID from a :processing line correctly" do
30
+ line = 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
31
+ @oink.should parse_line(line).as(:processing).and_capture(:pid => 67783, :controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET', :ip => '1.1.1.1')
32
+ end
33
+
34
+ it "should parse a :instance_type_counter correctly" do
35
+
36
+ line = "Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1"
37
+
38
+ @oink.should parse_line(line).as(:instance_type_counter).and_capture(:pid => 26364, :instance_counts => {'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1})
39
+ end
40
+ end
41
+
42
+ describe '#parse_io' do
43
+ context "Rails 2.2 style log" do
44
+ before(:each) do
45
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(
46
+ RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
47
+ end
48
+
49
+ it "should parse requests" do
50
+ request_counter.should_receive(:hit!).exactly(4).times
51
+
52
+ @log_parser.parse_file(log_fixture(:oink_22)) do |request|
53
+ request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
54
+ end
55
+ end
56
+
57
+ it "should not record :memory_diff on first request" do
58
+ @log_parser.parse_file(log_fixture(:oink_22)) do |request|
59
+ if @log_parser.parsed_requests == 1
60
+ request[:memory_diff].should == nil
61
+ end
62
+ end
63
+ end
64
+
65
+ it "should record :memory_diff of 2nd tracked PID" do
66
+ @log_parser.parse_file(log_fixture(:oink_22)) do |request|
67
+ if @log_parser.parsed_requests == 3
68
+ request[:memory_diff].should == 50000*1024
69
+ end
70
+ end
71
+ end
72
+
73
+ it "should record :memory_diff of 1st tracked PID" do
74
+ @log_parser.parse_file(log_fixture(:oink_22)) do |request|
75
+ if @log_parser.parsed_requests == 4
76
+ request[:memory_diff].should == 30000*1024
77
+ end
78
+ end
79
+ end
80
+ end
81
+
82
+ context 'Rails 2.2 style log w/failure' do
83
+ before(:each) do
84
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(
85
+ RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
86
+ end
87
+
88
+ it "should parse requests" do
89
+ request_counter.should_receive(:hit!).exactly(4).times
90
+
91
+ @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
92
+ request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
93
+ end
94
+ end
95
+
96
+ it "should ignore memory changes when a failure occurs" do
97
+ @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
98
+ if @log_parser.parsed_requests == 4
99
+ request[:memory_diff].should == nil
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
105
+ end
@@ -1,65 +1,43 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Postgresql do
4
4
 
5
- it "should be a valid file format" do
6
- RequestLogAnalyzer::FileFormat.load(:Postgresql).should be_valid
7
- end
5
+ subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
6
+ let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
7
+
8
+ it { should be_valid }
8
9
 
9
10
  describe '#parse_line' do
10
- before(:each) do
11
- @file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
12
- end
13
-
14
11
  it "should parse a :query line correctly" do
15
- line = '2004-05-07 11:58:36 LOG: query: SELECT plugin_id, plugin_name FROM plugins'
16
- @file_format.should parse_line(line).as(:query).and_capture(:timestamp => 20040507115836, :query_fragment => 'SELECT plugin_id, plugin_name FROM plugins')
12
+ line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
13
+ subject.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
17
14
  end
18
15
 
19
16
  it "should parse a :query_fragment line correctly" do
20
- line = ' groups.type_id,users.user_name,users.realname,'
21
- @file_format.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => "groups.type_id,users.user_name,users.realname,")
17
+ line = ' ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
18
+ subject.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
22
19
  end
23
20
 
24
- it "should parse a :duration line correctly" do
25
- line = '2004-05-07 11:58:36 LOG: duration: 0.002612 sec'
26
- @file_format.should parse_line(line).as(:duration).and_capture(:query_time => 0.002612)
27
- end
21
+ it "should parse a :query line correctly" do
22
+ line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
23
+ subject.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
24
+ end
28
25
  end
29
26
 
30
27
  describe '#parse_io' do
31
- before(:each) do
32
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
33
- end
34
-
35
28
  it "should parse a multiline query entry correctly" do
36
- fixture = <<EOS
37
- 2004-05-07 11:58:22 LOG: query: SELECT groups.group_name,groups.unix_group_name,
38
- groups.type_id,users.user_name,users.realname,
39
- news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details
40
- FROM users,news_bytes,groups
41
- WHERE news_bytes.group_id='98' AND news_bytes.is_approved <> '4'
42
- AND users.user_id=news_bytes.submitted_by
43
- AND news_bytes.group_id=groups.group_id
44
- ORDER BY post_date DESC LIMIT 10 OFFSET 0
45
- 2004-05-07 11:58:22 LOG: duration: 0.002680 sec
46
- EOS
47
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
48
- request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
49
- request[:query].should == "SELECT groups.group_name,groups.unix_group_name, groups.type_id,users.user_name,users.realname, news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details FROM users,news_bytes,groups WHERE news_bytes.group_id=:string AND news_bytes.is_approved <> :string AND users.user_id=news_bytes.submitted_by AND news_bytes.group_id=groups.group_id ORDER BY post_date DESC LIMIT :int OFFSET :int"
50
- end
51
- end
52
-
53
- it "should parse a dualline query entry correctly" do
54
- fixture = <<EOS
55
- 2004-05-07 11:58:36 LOG: query: SELECT type, count FROM project_sums_agg WHERE group_id=59
56
- 2004-05-07 11:58:36 LOG: duration: 0.001197 sec
57
- EOS
58
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
59
- request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
60
- request[:query].should == "SELECT type, count FROM project_sums_agg WHERE group_id=:int"
29
+ fixture = <<-EOS
30
+ 2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG: 00000: duration: 0.195 ms statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {}
31
+ ', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id"
32
+ 2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081
33
+ EOS
34
+
35
+ request_counter.should_receive(:hit!).exactly(1).times
36
+ log_parser.should_not_receive(:warn)
37
+ log_parser.parse_io(StringIO.new(fixture)) do |request|
38
+ request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Postgresql::Request) && request.completed?
39
+ request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
61
40
  end
62
41
  end
63
42
  end
64
43
  end
65
-
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Rack do
4
4
 
@@ -10,30 +10,57 @@ describe RequestLogAnalyzer::FileFormat::Rails do
10
10
  before(:each) { @file_format = RequestLogAnalyzer::FileFormat.load(:rails3) }
11
11
 
12
12
  it "should parse :started lines correctly" do
13
- line = 'Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18'
13
+ line = 'Started GET "/queries" for 127.0.0.1 at Thu Feb 25 16:15:18 -0800 2010'
14
14
  @file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
15
15
  :path => '/queries', :ip => '127.0.0.1', :timestamp => 20100225161518)
16
16
  end
17
17
 
18
+ it "should parse :started lines in Oct, Nov and Dec correctly" do
19
+ line = 'Started GET "/queries" for 127.0.0.1 at Thu Oct 25 16:15:18 -0800 2010'
20
+ @file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
21
+ :path => '/queries', :ip => '127.0.0.1', :timestamp => 20101025161518)
22
+ end
23
+
24
+ it "should parse :started lines in Ruby 1.9.2 format correctly" do
25
+ line = 'Started GET "/queries" for 127.0.0.1 at 2010-10-26 02:27:15 +0000'
26
+ @file_format.should parse_line(line).as(:started).and_capture(:method => 'GET',
27
+ :path => '/queries', :ip => '127.0.0.1', :timestamp => 20101026022715)
28
+ end
29
+
18
30
  it "should parse :processing lines correctly" do
19
31
  line = ' Processing by QueriesController#index as HTML'
20
32
  @file_format.should parse_line(line).as(:processing).and_capture(
21
33
  :controller => 'QueriesController', :action => 'index', :format => 'HTML')
22
34
  end
23
-
24
-
25
- # it "should parse beta :completed lines correctly" do
26
- # line = 'Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200'
27
- # @file_format.should parse_line(line).as(:completed).and_capture(
28
- # :duration => 0.009, :status => 200)
29
- # end
35
+ it "should parse nested :processing lines correctly" do
36
+ line = ' Processing by Projects::QueriesController#index as HTML'
37
+ @file_format.should parse_line(line).as(:processing).and_capture(
38
+ :controller => 'Projects::QueriesController', :action => 'index', :format => 'HTML')
39
+ end
40
+ it "should parse :processing lines correctly with format */*" do
41
+ line = ' Processing by ProjectsController#avatar as */*'
42
+ @file_format.should parse_line(line).as(:processing).and_capture(
43
+ :controller => 'ProjectsController', :action => 'avatar', :format => '*/*')
44
+ end
30
45
 
31
46
  it "should parse :completed lines correctly" do
32
- line = 'Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)'
47
+ line = 'Completed 200 OK in 170ms (Views: 78.0ms | ActiveRecord: 48.2ms)'
33
48
  @file_format.should parse_line(line).as(:completed).and_capture(
34
- :duration => 0.170, :status => 200)
49
+ :duration => 0.170, :view => 0.078, :db => 0.0482, :status => 200)
50
+ end
51
+
52
+ it "should parse :completed lines correctly when ActiveRecord is not mentioned" do
53
+ line = 'Completed 200 OK in 364ms (Views: 31.4ms)'
54
+ @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.364, :status => 200)
35
55
  end
36
56
 
57
+ it "should parse :completed lines correctly when other durations are specified" do
58
+ line = 'Completed 200 OK in 384ms (Views: 222.0ms | ActiveRecord: 121.0ms | Sphinx: 0.0ms)'
59
+ @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.384, :view => 0.222,
60
+ :db => 0.121, :status => 200)
61
+ end
62
+
63
+
37
64
  it "should pase :failure lines correctly" do
38
65
  line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"
39
66
  @file_format.should parse_line(line).as(:failure).and_capture(:line => 3,
@@ -50,7 +77,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
50
77
 
51
78
  it "should parse a successful request correctly" do
52
79
  log = <<-EOLOG
53
- Started GET "/" for 127.0.0.1 at 2010-03-19 06:40:41
80
+ Started GET "/" for 127.0.0.1 at Fri Mar 19 06:40:41 -0700 2010
54
81
  Processing by QueriesController#index as HTML
55
82
  SQL (16.3ms) SHOW TABLES
56
83
  Query Load (32.0ms) SELECT `queries`.* FROM `queries`
@@ -68,7 +95,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
68
95
 
69
96
  it "should parse an unroutable request correctly" do
70
97
  log = <<-EOLOG
71
- Started GET "/404" for 127.0.0.1 at 2010-03-19 06:40:57
98
+ Started GET "/404" for 127.0.0.1 at Fri Mar 19 06:40:57 -0700 2010
72
99
 
73
100
  ActionController::RoutingError (No route matches "/404"):
74
101
 
@@ -87,7 +114,7 @@ describe RequestLogAnalyzer::FileFormat::Rails do
87
114
 
88
115
  it "should parse a failing request correctly" do
89
116
  log = <<-EOLOG
90
- Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at 2010-03-01 18:44:33
117
+ Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at Mon Mar 01 18:44:33 -0800 2010
91
118
  Processing by QueriesController#execute as CSV
92
119
  Parameters: {"commit"=>"Run query", "authenticity_token"=>"pz9WcxkcrlG/43eg6BgSAnJL7yIsaffuHbYxPHUsUzQ=", "id"=>"397638749"}
93
120
 
@@ -117,4 +144,4 @@ describe RequestLogAnalyzer::FileFormat::Rails do
117
144
  end
118
145
  end
119
146
  end
120
- end
147
+ end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Rails do
4
4
 
@@ -75,6 +75,16 @@ describe RequestLogAnalyzer::FileFormat::Rails do
75
75
  @rails.should parse_line(line).as(:completed).and_capture(:duration => 0.614, :db => 0.031, :view => 0.120, :status => 200, :url => 'http://floorplanner.local/demo')
76
76
  end
77
77
 
78
+ it "should parse a Rails 2.2 style :completed line correctly when AR is disabled" do
79
+ line = prefix + 'Completed in 597ms (View: 298 | 200 OK [http://shapado.com]'
80
+ @rails.should parse_line(line).as(:completed).and_capture(:duration => 0.597, :db => nil, :view => 0.298, :status => 200, :url => 'http://shapado.com')
81
+ end
82
+
83
+ it "should parse a Rails 2.2 style :completed line without view" do
84
+ line = prefix + "Completed in 148ms (DB: 0) | 302 Found [http://iwp-sod.hargray.org/login]"
85
+ @rails.should parse_line(line).as(:completed).and_capture(:duration => 0.148, :db => 0.0, :view => nil, :status => 302, :url => 'http://iwp-sod.hargray.org/login')
86
+ end
87
+
78
88
  it "should parse a :failure line with exception correctly" do
79
89
  line = prefix + "NoMethodError (undefined method `update_domain_account' for nil:NilClass):"
80
90
  @rails.should parse_line(line).as(:failure).and_capture(:error => 'NoMethodError', :message => "undefined method `update_domain_account' for nil:NilClass")
@@ -0,0 +1,46 @@
1
+ require 'spec_helper'
2
+
3
+ describe RequestLogAnalyzer::FileFormat::W3c do
4
+
5
+ before(:each) do
6
+ @file_format = RequestLogAnalyzer::FileFormat.load(:w3c)
7
+ @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
8
+ # date time c-ip cs-username s-ip s-port cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(User-Agent) cs(Referrer)
9
+ @sample = '2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/'
10
+ end
11
+
12
+ it "should be a valid file format" do
13
+ @file_format.should be_valid
14
+ end
15
+
16
+ it "should parse access lines and capture all of its fields" do
17
+ @file_format.should have_line_definition(:access).capturing(:timestamp, :remote_ip, :username, :local_ip, :port, :method, :path, :http_status, :bytes_sent, :bytes_received, :duration, :user_agent, :referer)
18
+ end
19
+
20
+ it "should match the sample line" do
21
+ @file_format.parse_line(@sample).should include(:line_definition, :captures)
22
+ end
23
+
24
+ it "should not match a nonsense line" do
25
+ @file_format.parse_line('#Software: Microsoft Internet Information Services 6.0').should be_nil
26
+ end
27
+
28
+ it "should parse and convert the sample fields correctly" do
29
+ @log_parser.parse_io(StringIO.new(@sample)) do |request|
30
+ request[:timestamp].should == 20020524201801
31
+ request[:remote_ip].should == "172.224.24.114"
32
+ request[:username].should == nil
33
+ request[:local_ip].should == "206.73.118.24"
34
+ request[:port].should == 80
35
+ request[:method].should == 'GET'
36
+ request[:path].should == '/Default.htm'
37
+ request[:http_status].should == 200
38
+ request[:bytes_sent].should == 7930
39
+ request[:bytes_received].should == 248
40
+ request[:duration].should == 0.031
41
+ request[:user_agent].should == 'Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server)'
42
+ request[:referer].should == 'http://64.224.24.114/'
43
+ end
44
+ end
45
+
46
+ end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Filter::Base, 'base filter' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Mailer, 'mailer' do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Request do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Source::LogParser, :requests do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::Duration do
4
4
 
@@ -27,7 +27,9 @@ describe RequestLogAnalyzer::Tracker::Duration do
27
27
  it "should generate a YAML output" do
28
28
  @tracker.update(request(:category => 'a', :duration => 0.2))
29
29
  @tracker.update(request(:category => 'b', :duration => 0.2))
30
- @tracker.to_yaml_object.should == {"a"=>{:hits=>1, :min=>0.2, :mean=>0.2, :max=>0.2, :sum_of_squares=>0.0, :sum=>0.2}, "b"=>{:hits=>1, :min=>0.2, :mean=>0.2, :max=>0.2, :sum_of_squares=>0.0, :sum=>0.2}}
30
+ @tracker.to_yaml_object.keys.should =~ ['a', 'b']
31
+ @tracker.to_yaml_object['a'].should include(:min => 0.2, :hits => 1, :max => 0.2, :mean => 0.2, :sum => 0.2, :sum_of_squares => 0.0)
32
+ @tracker.to_yaml_object['b'].should include(:min => 0.2, :hits => 1, :max => 0.2, :mean => 0.2, :sum => 0.2, :sum_of_squares => 0.0)
31
33
  end
32
34
  end
33
35
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::Frequency do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::HourlySpread do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::NumericValue do
4
4
 
@@ -22,7 +22,7 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
22
22
  @tracker.categories['b'][:sum].should == 4
23
23
  end
24
24
  end
25
-
25
+
26
26
 
27
27
  context 'dynamic category' do
28
28
  before(:each) do
@@ -40,11 +40,12 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
40
40
  @tracker.categories['slow'][:sum].should == 4
41
41
  end
42
42
  end
43
-
44
- describe '#update' do
45
43
 
44
+ describe '#update' do
46
45
  before(:each) do
47
- @tracker = RequestLogAnalyzer::Tracker::NumericValue.new(:value => :duration, :category => :category)
46
+ @tracker = RequestLogAnalyzer::Tracker::NumericValue.new(:value => :duration, :category => :category,
47
+ :min_bucket_value => 0.0001, :max_bucket_value => 1000)
48
+
48
49
  @tracker.prepare
49
50
 
50
51
  @tracker.update(request(:category => 'a', :duration => 0.4))
@@ -53,11 +54,11 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
53
54
  end
54
55
 
55
56
  it "should sum of the durations for a category correctly" do
56
- @tracker.sum('a').should be_close(0.9, 0.000001)
57
+ @tracker.sum('a').should be_within(0.000001).of(0.9)
57
58
  end
58
59
 
59
60
  it "should overall sum of the durations correctly" do
60
- @tracker.sum_overall.should be_close(0.9, 0.000001)
61
+ @tracker.sum_overall.should be_within(0.000001).of(0.9)
61
62
  end
62
63
 
63
64
  it "should keep track of the minimum and maximum duration" do
@@ -66,21 +67,39 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
66
67
  end
67
68
 
68
69
  it "should calculate the mean duration correctly" do
69
- @tracker.mean('a').should be_close(0.3, 0.000001)
70
+ @tracker.mean('a').should be_within(0.000001).of(0.3)
70
71
  end
71
72
 
72
73
  it "should calculate the overall mean duration correctly" do
73
- @tracker.mean_overall.should be_close(0.3, 0.000001)
74
+ @tracker.mean_overall.should be_within(0.000001).of(0.3)
74
75
  end
75
76
 
76
77
  it "should calculate the duration variance correctly" do
77
- @tracker.variance('a').should be_close(0.01, 0.000001)
78
+ @tracker.variance('a').should be_within(0.000001).of(0.01)
78
79
  end
79
80
 
80
81
  it "should calculate the duration standard deviation correctly" do
81
- @tracker.stddev('a').should be_close(0.1, 0.000001)
82
+ @tracker.stddev('a').should be_within(0.000001).of(0.1)
83
+ end
84
+
85
+ it "should calculate the bucket spread correctly" do
86
+ @tracker.update(request(:category => 'a', :duration => 0.3))
87
+ @tracker.update(request(:category => 'a', :duration => 0.3))
88
+ @tracker.update(request(:category => 'a', :duration => 0.3))
89
+ @tracker.update(request(:category => 'a', :duration => 0.3))
90
+ @tracker.update(request(:category => 'a', :duration => 0.3))
91
+ @tracker.update(request(:category => 'a', :duration => 0.3))
92
+ @tracker.update(request(:category => 'a', :duration => 0.3))
93
+ # 0.2, 0.3 and 0.4 are already there, so, 10 values in total
94
+
95
+ @tracker.median('a').should be_within(0.01).of(0.3)
96
+
97
+ @tracker.percentile_interval('a', 80).begin.should be_within(0.01).of(0.3)
98
+ @tracker.percentile_interval('a', 80).end.should be_within(0.01).of(0.3)
99
+ @tracker.percentile_interval('a', 90).begin.should be_within(0.01).of(0.2)
100
+ @tracker.percentile_interval('a', 90).end.should be_within(0.01).of(0.4)
82
101
  end
83
- end
102
+ end
84
103
 
85
104
  describe '#report' do
86
105
  before(:each) do
@@ -113,9 +132,13 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
113
132
  it "should generate a YAML output" do
114
133
  @tracker.update(request(:category => 'a', :blah => 2))
115
134
  @tracker.update(request(:category => 'b', :blah => 3))
116
- @tracker.to_yaml_object.should == {
117
- "a" => { :min => 2, :hits => 1, :max => 2, :mean => 2.0, :sum => 2, :sum_of_squares => 0.0 },
118
- "b" => { :min => 3, :hits => 1, :max => 3, :mean => 3.0, :sum => 3, :sum_of_squares => 0.0 }}
135
+ @tracker.to_yaml_object.keys.should =~ ['a', 'b']
136
+
137
+ @tracker.to_yaml_object['a'].should include(:min => 2, :hits => 1, :max => 2, :mean => 2.0, :sum => 2, :sum_of_squares => 0.0)
138
+ @tracker.to_yaml_object['a'][:interval_95_percent].should be_member(2)
139
+
140
+ @tracker.to_yaml_object['b'].should include(:min => 3, :hits => 1, :max => 3, :mean => 3.0, :sum => 3, :sum_of_squares => 0.0)
141
+ @tracker.to_yaml_object['b'][:interval_95_percent].should be_member(3)
119
142
  end
120
143
  end
121
144
 
@@ -161,6 +184,4 @@ describe RequestLogAnalyzer::Tracker::NumericValue do
161
184
  @tracker.display_value(9000_000_001_001_000).should eql('9000T')
162
185
  end
163
186
  end
164
-
165
-
166
187
  end
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::Timespan do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::Base do
4
4
 
@@ -1,4 +1,4 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Tracker::Traffic do
4
4