request-log-analyzer 1.9.10 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. data/.gitignore +4 -1
  2. data/.infinity_test +8 -0
  3. data/Gemfile +2 -0
  4. data/bin/request-log-analyzer +0 -1
  5. data/lib/request_log_analyzer/aggregator/database_inserter.rb +2 -1
  6. data/lib/request_log_analyzer/aggregator.rb +5 -5
  7. data/lib/request_log_analyzer/controller.rb +0 -3
  8. data/lib/request_log_analyzer/database.rb +6 -7
  9. data/lib/request_log_analyzer/file_format/apache.rb +19 -28
  10. data/lib/request_log_analyzer/file_format/delayed_job2.rb +2 -2
  11. data/lib/request_log_analyzer/file_format/delayed_job21.rb +2 -2
  12. data/lib/request_log_analyzer/file_format/haproxy.rb +232 -0
  13. data/lib/request_log_analyzer/file_format/mysql.rb +5 -5
  14. data/lib/request_log_analyzer/file_format/rails3.rb +9 -2
  15. data/lib/request_log_analyzer/file_format/w3c.rb +56 -0
  16. data/lib/request_log_analyzer/file_format.rb +46 -15
  17. data/lib/request_log_analyzer/filter.rb +4 -5
  18. data/lib/request_log_analyzer/line_definition.rb +6 -4
  19. data/lib/request_log_analyzer/output.rb +3 -5
  20. data/lib/request_log_analyzer/request.rb +7 -0
  21. data/lib/request_log_analyzer/source/log_parser.rb +56 -4
  22. data/lib/request_log_analyzer/source.rb +3 -4
  23. data/lib/request_log_analyzer/tracker.rb +8 -8
  24. data/lib/request_log_analyzer.rb +15 -29
  25. data/request-log-analyzer.gemspec +4 -4
  26. data/spec/fixtures/mysql_slow_query.log +0 -1
  27. data/spec/integration/command_line_usage_spec.rb +0 -5
  28. data/spec/lib/helpers.rb +2 -2
  29. data/spec/lib/matchers.rb +38 -7
  30. data/spec/lib/mocks.rb +1 -5
  31. data/spec/spec_helper.rb +3 -2
  32. data/spec/unit/database/base_class_spec.rb +1 -0
  33. data/spec/unit/file_format/amazon_s3_format_spec.rb +58 -55
  34. data/spec/unit/file_format/apache_format_spec.rb +90 -161
  35. data/spec/unit/file_format/common_regular_expressions_spec.rb +51 -26
  36. data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -31
  37. data/spec/unit/file_format/delayed_job2_format_spec.rb +27 -32
  38. data/spec/unit/file_format/delayed_job_format_spec.rb +44 -63
  39. data/spec/unit/file_format/haproxy_format_spec.rb +82 -0
  40. data/spec/unit/file_format/line_definition_spec.rb +26 -33
  41. data/spec/unit/file_format/merb_format_spec.rb +22 -37
  42. data/spec/unit/file_format/mysql_format_spec.rb +80 -123
  43. data/spec/unit/file_format/oink_format_spec.rb +29 -61
  44. data/spec/unit/file_format/postgresql_format_spec.rb +10 -21
  45. data/spec/unit/file_format/rack_format_spec.rb +49 -44
  46. data/spec/unit/file_format/rails3_format_spec.rb +46 -38
  47. data/spec/unit/file_format/rails_format_spec.rb +52 -68
  48. data/spec/unit/file_format/w3c_format_spec.rb +47 -0
  49. data/spec/unit/source/log_parser_spec.rb +1 -1
  50. metadata +12 -7
  51. data/lib/mixins/gets_memory_protection.rb +0 -80
  52. data/lib/request_log_analyzer/output/fancy_html.rb +0 -44
  53. data/lib/request_log_analyzer/source/database_loader.rb +0 -87
@@ -2,153 +2,110 @@ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Mysql do
4
4
 
5
- it "should be a valid file format" do
6
- RequestLogAnalyzer::FileFormat.load(:mysql).should be_valid
7
- end
8
-
5
+ subject { RequestLogAnalyzer::FileFormat.load(:mysql) }
6
+
7
+ it { should be_well_formed }
8
+ it { should have_line_definition(:time).capturing(:timestamp) }
9
+ it { should have_line_definition(:user_host).capturing(:user, :host, :ip) }
10
+ it { should have_line_definition(:query_statistics).capturing(:query_time, :lock_time, :rows_sent, :rows_examined) }
11
+ it { should have_line_definition(:use_database).capturing(:database) }
12
+ it { should have_line_definition(:query_part).capturing(:query_fragment) }
13
+ it { should have_line_definition(:query).capturing(:query) }
14
+ it { should have(7).report_trackers }
15
+
9
16
  describe '#parse_line' do
10
- before(:each) do
11
- @file_format = RequestLogAnalyzer::FileFormat.load(:mysql)
12
- end
13
-
14
- it "should parse a :time line correctly" do
15
- line = '# Time: 091112 8:13:56'
16
- @file_format.should parse_line(line).as(:time).and_capture(:timestamp => 20091112081356)
17
- end
18
-
19
- it "should parse a :user_host line correctly with IP present" do
20
- line = '# User@Host: admin[admin] @ db1 [10.0.0.1]'
21
- @file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => 'db1', :ip => '10.0.0.1')
22
- end
23
-
24
- it "should parse a :user_host line correctly without a host" do
25
- line = '# User@Host: admin[admin] @ [10.0.0.1]'
26
- @file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => '', :ip => '10.0.0.1')
27
- end
28
-
29
- it "should parse a :user_host line correctly with IP absent" do
30
- line = '# User@Host: root[root] @ localhost []'
31
- @file_format.should parse_line(line).as(:user_host).and_capture(:user => "root", :host => 'localhost', :ip => "")
32
- end
33
-
34
- it "should parse a :query_statistics line" do
35
- line = '# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307'
36
- @file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
37
- :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
38
- end
39
-
40
- it "should parse a :query_statistics line with floating point durations" do
41
- line = '# Query_time: 10.00000 Lock_time: 0.00000 Rows_sent: 1191307 Rows_examined: 1191307'
42
- @file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
43
- :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
44
- end
45
-
46
- it "should parse a :query_part line" do
47
- line = ' AND clients.index > 0'
48
- @file_format.should parse_line(line).as(:query_part).and_capture(:query_fragment => line)
49
- end
50
17
 
51
- it "should parse a final :query line" do
52
- line = 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`; '
53
- @file_format.should parse_line(line).as(:query).and_capture(:query =>
54
- 'SELECT /*!:int SQL_NO_CACHE */ * FROM events')
55
- end
56
-
57
- it "should parse a :use_database line" do
58
- line = 'use db;'
59
- @file_format.should parse_line(line).as(:use_database).and_capture(:database => 'db')
60
- end
61
-
62
- it "should not parse a SET timestamp line" do
63
- line = 'SET timestamp=1250651725;'
64
- @file_format.should_not parse_line(line)
65
- end
66
-
67
- it "should not parse a SET insert_id line" do
68
- line = 'SET insert_id=1250651725;'
69
- @file_format.should_not parse_line(line)
70
- end
71
-
72
- it "should not parse a SET timestamp, insert_id line" do
73
- line = 'SET timestamp=1250651725, insert_id=45674;'
74
- @file_format.should_not parse_line(line)
75
- end
76
-
77
-
18
+ let(:time_sample) { '# Time: 091112 8:13:56' }
19
+ let(:user_host_sample) { '# User@Host: admin[admin] @ db1 [10.0.0.1]' }
20
+ let(:user_host_wo_host_sample) { '# User@Host: admin[admin] @ [10.0.0.1]' }
21
+ let(:user_host_wo_ip_sample) { '# User@Host: root[root] @ localhost []' }
22
+ let(:float_query_statistics_sample) { '# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307' }
23
+ let(:int_query_statistics_sample) { '# Query_time: 10.00000 Lock_time: 0.00000 Rows_sent: 1191307 Rows_examined: 1191307' }
24
+ let(:partial_query_sample) { 'AND clients.index > 0' }
25
+ let(:full_query_sample) { 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`; ' }
26
+ let(:use_db_sample) { 'use db;' }
27
+ let(:set_timestamp_sample) { 'SET timestamp=1250651725;' }
28
+ let(:set_insertid_sample) { 'SET insert_id=1250651725;' }
29
+ let(:set_timestamp_insertid_sample) { 'SET timestamp=1250651725, insert_id=45674;' }
30
+
31
+ it { should parse_line(time_sample).as(:time).and_capture(:timestamp => 20091112081356) }
32
+ it { should parse_line(user_host_sample).as(:user_host).and_capture(:user => "admin", :host => 'db1', :ip => '10.0.0.1') }
33
+ it { should parse_line(user_host_wo_host_sample, 'without host').as(:user_host).and_capture(:user => "admin", :host => '', :ip => '10.0.0.1') }
34
+ it { should parse_line(user_host_wo_ip_sample, 'without IP').as(:user_host).and_capture(:user => "root", :host => 'localhost', :ip => "") }
35
+ it { should parse_line(float_query_statistics_sample, 'using floats').as(:query_statistics).and_capture(:query_time => 10.0, :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307) }
36
+ it { should parse_line(int_query_statistics_sample, 'using integers').as(:query_statistics).and_capture(:query_time => 10.0, :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307) }
37
+ it { should parse_line(partial_query_sample).as(:query_part).and_capture(:query_fragment => 'AND clients.index > 0') }
38
+ it { should parse_line(full_query_sample).as(:query).and_capture(:query => 'SELECT /*!:int SQL_NO_CACHE */ * FROM events') }
39
+ it { should parse_line(use_db_sample).as(:use_database).and_capture(:database => 'db') }
40
+
41
+ it { should_not parse_line(set_timestamp_sample) }
42
+ it { should_not parse_line(set_insertid_sample) }
43
+ it { should_not parse_line(set_timestamp_insertid_sample) }
78
44
  end
79
45
 
80
46
  describe '#parse_io' do
81
- before(:each) do
82
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:mysql))
83
- end
47
+ let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
84
48
 
85
49
  it "should parse a single line query entry correctly" do
86
- fixture = <<EOS
87
- # Time: 091112 18:13:56
88
- # User@Host: admin[admin] @ db1 [10.0.0.1]
89
- # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
90
- SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
91
- EOS
92
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
93
- request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
50
+ fixture = <<-EOS
51
+ # Time: 091112 18:13:56
52
+ # User@Host: admin[admin] @ db1 [10.0.0.1]
53
+ # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
54
+ SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
55
+ EOS
56
+
57
+ log_parser.parse_string(fixture) do |request|
94
58
  request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
95
59
  end
96
60
  end
97
61
 
98
62
  it "should parse a multiline query entry correctly" do
99
- fixture = <<EOS
100
- # Time: 091112 18:13:56
101
- # User@Host: admin[admin] @ db1 [10.0.0.1]
102
- # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
103
- SELECT * FROM `clients` WHERE (1=1
104
- AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
105
- AND clients.index > 0
106
- ) AND (clients.deleted_at IS NULL);
107
- EOS
108
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
109
- request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
63
+ fixture = <<-EOS
64
+ # Time: 091112 18:13:56
65
+ # User@Host: admin[admin] @ db1 [10.0.0.1]
66
+ # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
67
+ SELECT * FROM `clients` WHERE (1=1
68
+ AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
69
+ AND clients.index > 0
70
+ ) AND (clients.deleted_at IS NULL);
71
+ EOS
72
+
73
+ log_parser.parse_string(fixture) do |request|
110
74
  request[:query].should == "SELECT * FROM clients WHERE (:int=:int AND clients.valid_from < :date AND (clients.valid_to IS NULL or clients.valid_to > :date) AND clients.index > :int ) AND (clients.deleted_at IS NULL)"
111
75
  end
112
76
  end
113
77
 
114
- it "should parse a request without timestamp correctly" do
115
- fixture = <<EOS
116
- # User@Host: admin[admin] @ db1 [10.0.0.1]
117
- # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
118
- SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
119
- EOS
120
- request_counter.should_receive(:hit!).once
121
- @log_parser.should_not_receive(:warn)
122
-
123
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
124
- request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
125
- end
78
+ it "should parse a request without timestamp correctly, without warnings" do
79
+ fixture = <<-EOS
80
+ # User@Host: admin[admin] @ db1 [10.0.0.1]
81
+ # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
82
+ SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
83
+ EOS
84
+
85
+ log_parser.should_receive(:handle_request).once
86
+ log_parser.should_not_receive(:warn)
87
+ log_parser.parse_string(fixture)
126
88
  end
127
89
 
128
90
  it "should parse a query with context information correctly" do
129
- fixture = <<EOS
130
- # Time: 091112 18:13:56
131
- # User@Host: admin[admin] @ db1 [10.0.0.1]
132
- # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
133
- use database_name;
134
- SET timestamp=4324342342423, insert_id = 224253443;
135
- SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
136
- EOS
137
-
138
- request_counter.should_receive(:hit!).once
139
- @log_parser.should_not_receive(:warn)
140
-
141
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
142
- request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
91
+ fixture = <<-EOS
92
+ # Time: 091112 18:13:56
93
+ # User@Host: admin[admin] @ db1 [10.0.0.1]
94
+ # Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
95
+ use database_name;
96
+ SET timestamp=4324342342423, insert_id = 224253443;
97
+ SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
98
+ EOS
99
+
100
+ log_parser.parse_string(fixture) do |request|
143
101
  request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
144
102
  end
145
-
146
103
  end
147
104
 
148
-
149
105
  it "should find 26 completed sloq query entries" do
150
- @log_parser.should_receive(:handle_request).exactly(26).times
151
- @log_parser.parse_file(log_fixture(:mysql_slow_query))
106
+ log_parser.should_not_receive(:warn)
107
+ log_parser.should_receive(:handle_request).exactly(26).times
108
+ log_parser.parse_file(log_fixture(:mysql_slow_query))
152
109
  end
153
110
  end
154
111
  end
@@ -1,103 +1,71 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Oink do
4
- describe '.create' do
5
4
 
6
- context 'without providing a lines argument' do
7
- before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink) }
8
-
9
- it "should create a valid file format" do
10
- @oink.should be_valid
11
- end
12
-
13
- it "should parse :memory_usage line" do
14
- @oink.line_definitions.should include(:memory_usage)
15
- end
16
-
17
- end
18
-
19
- end
5
+ subject { RequestLogAnalyzer::FileFormat.load(:oink) }
20
6
 
7
+ it { should have_line_definition(:memory_usage).capturing(:pid, :memory) }
8
+ it { should have_line_definition(:processing).capturing(:pid, :controller, :action, :ip) }
9
+ it { should have_line_definition(:instance_type_counter).capturing(:pid, :instance_counts) }
10
+ it { should have(12).report_trackers }
11
+
21
12
  describe '#parse_line' do
22
- before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink, :all) }
23
-
13
+ let(:memory_usage_sample) { 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783' }
14
+ let(:processing_sample) { 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]' }
15
+ let(:instance_type_counter_sample) { "Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1" }
16
+
24
17
  it "should parse a :memory_usage line correctly" do
25
- line = 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783'
26
- @oink.should parse_line(line).as(:memory_usage).and_capture(:pid => 67783, :memory => 714052)
18
+ subject.should parse_line(memory_usage_sample).as(:memory_usage).and_capture(:pid => 67783, :memory => 714052)
27
19
  end
28
20
 
29
21
  it "should parse the PID from a :processing line correctly" do
30
- line = 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
31
- @oink.should parse_line(line).as(:processing).and_capture(:pid => 67783, :controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET', :ip => '1.1.1.1')
22
+ subject.should parse_line(processing_sample).as(:processing).and_capture(:pid => 67783, :controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET', :ip => '1.1.1.1')
32
23
  end
33
24
 
34
25
  it "should parse a :instance_type_counter correctly" do
35
-
36
- line = "Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1"
37
-
38
- @oink.should parse_line(line).as(:instance_type_counter).and_capture(:pid => 26364, :instance_counts => {'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1})
26
+ subject.should parse_line(instance_type_counter_sample).as(:instance_type_counter).and_capture(:pid => 26364, :instance_counts => {'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1})
39
27
  end
40
28
  end
41
29
 
42
30
  describe '#parse_io' do
31
+ let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
32
+
43
33
  context "Rails 2.2 style log" do
44
- before(:each) do
45
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(
46
- RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
47
- end
48
-
49
34
  it "should parse requests" do
50
- request_counter.should_receive(:hit!).exactly(4).times
51
-
52
- @log_parser.parse_file(log_fixture(:oink_22)) do |request|
53
- request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
54
- end
35
+ log_parser.should_receive(:handle_request).exactly(4).times
36
+ log_parser.should_not_receive(:warn)
37
+ log_parser.parse_file(log_fixture(:oink_22))
55
38
  end
56
39
 
57
40
  it "should not record :memory_diff on first request" do
58
- @log_parser.parse_file(log_fixture(:oink_22)) do |request|
59
- if @log_parser.parsed_requests == 1
60
- request[:memory_diff].should == nil
61
- end
41
+ log_parser.parse_file(log_fixture(:oink_22)) do |request|
42
+ request[:memory_diff].should == nil if log_parser.parsed_requests == 1
62
43
  end
63
44
  end
64
45
 
65
46
  it "should record :memory_diff of 2nd tracked PID" do
66
- @log_parser.parse_file(log_fixture(:oink_22)) do |request|
67
- if @log_parser.parsed_requests == 3
68
- request[:memory_diff].should == 50000*1024
69
- end
47
+ log_parser.parse_file(log_fixture(:oink_22)) do |request|
48
+ request[:memory_diff].should == 50000 * 1024 if log_parser.parsed_requests == 3
70
49
  end
71
50
  end
72
51
 
73
52
  it "should record :memory_diff of 1st tracked PID" do
74
- @log_parser.parse_file(log_fixture(:oink_22)) do |request|
75
- if @log_parser.parsed_requests == 4
76
- request[:memory_diff].should == 30000*1024
77
- end
53
+ log_parser.parse_file(log_fixture(:oink_22)) do |request|
54
+ request[:memory_diff].should == 30000 * 1024 if log_parser.parsed_requests == 4
78
55
  end
79
56
  end
80
57
  end
81
58
 
82
59
  context 'Rails 2.2 style log w/failure' do
83
- before(:each) do
84
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(
85
- RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
86
- end
87
-
88
60
  it "should parse requests" do
89
- request_counter.should_receive(:hit!).exactly(4).times
90
-
91
- @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
92
- request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
93
- end
61
+ log_parser.should_receive(:handle_request).exactly(4).times
62
+ log_parser.should_not_receive(:warn)
63
+ log_parser.parse_file(log_fixture(:oink_22_failure))
94
64
  end
95
65
 
96
66
  it "should ignore memory changes when a failure occurs" do
97
- @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
98
- if @log_parser.parsed_requests == 4
99
- request[:memory_diff].should == nil
100
- end
67
+ log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
68
+ request[:memory_diff].should == nil if log_parser.parsed_requests == 4
101
69
  end
102
70
  end
103
71
  end
@@ -2,36 +2,29 @@ require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Postgresql do
4
4
 
5
- it "should be a valid file format" do
6
- RequestLogAnalyzer::FileFormat.load(:Postgresql).should be_valid
7
- end
5
+ subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
6
+ let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
7
+
8
+ it { should be_well_formed }
8
9
 
9
10
  describe '#parse_line' do
10
- before(:each) do
11
- @file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
12
- end
13
-
14
11
  it "should parse a :query line correctly" do
15
12
  line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
16
- @file_format.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
13
+ subject.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
17
14
  end
18
15
 
19
16
  it "should parse a :query_fragment line correctly" do
20
17
  line = ' ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
21
- @file_format.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
18
+ subject.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
22
19
  end
23
20
 
24
21
  it "should parse a :query line correctly" do
25
22
  line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
26
- @file_format.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
27
- end
23
+ subject.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
24
+ end
28
25
  end
29
26
 
30
27
  describe '#parse_io' do
31
- before(:each) do
32
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
33
- end
34
-
35
28
  it "should parse a multiline query entry correctly" do
36
29
  fixture = <<-EOS
37
30
  2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG: 00000: duration: 0.195 ms statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {}
@@ -39,14 +32,10 @@ describe RequestLogAnalyzer::FileFormat::Postgresql do
39
32
  2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081
40
33
  EOS
41
34
 
42
- request_counter.should_receive(:hit!).exactly(1).times
43
- @log_parser.should_not_receive(:warn)
44
- @log_parser.parse_io(StringIO.new(fixture)) do |request|
45
- request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Postgresql::Request) && request.completed?
35
+ log_parser.should_not_receive(:warn)
36
+ log_parser.parse_string(fixture) do |request|
46
37
  request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
47
38
  end
48
39
  end
49
-
50
40
  end
51
41
  end
52
-
@@ -1,50 +1,55 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::Rack do
4
+
5
+ subject { RequestLogAnalyzer::FileFormat.load(:rack)}
6
+ let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
7
+
8
+ it { should be_well_formed }
9
+ it { should have_line_definition(:access).capturing(:remote_host, :user, :remote_logname,
10
+ :timestamp, :http_method, :path, :http_version, :http_status, :bytes_sent, :duration) }
11
+
12
+ it { should have(7).report_trackers }
13
+
14
+ let(:sample1) { '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024' }
15
+ let(:sample2) { '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453' }
16
+ let(:sample3) { '127.0.0.1 - - [01/Oct/2009 07:58:10] "GET / HTTP/1.1" 200 1 0.0045' }
17
+ let(:irrelevant) { '== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel' }
18
+
19
+ describe '#parse_line' do
20
+
21
+ it { should parse_line(sample1, 'a sample access line').and_capture(
22
+ :remote_host => '127.0.0.1', :timestamp => 20091123214747, :user => nil,
23
+ :http_status => 200, :http_method => 'GET', :http_version => '1.1',
24
+ :duration => 0.0024, :bytes_sent => 3782, :remote_logname => nil,
25
+ :path => '/css/stylesheet.css')
26
+ }
27
+
28
+ it { should parse_line(sample2, 'another sample access line').and_capture(
29
+ :remote_host => '127.0.0.1', :timestamp => 20090916074008, :user => nil,
30
+ :http_status => 500, :http_method => 'GET', :http_version => '1.1',
31
+ :duration => 0.0453, :bytes_sent => 63183, :remote_logname => nil,
32
+ :path => '/favicon.ico')
33
+ }
34
+
35
+ it { should parse_line(sample3, 'a third sample access line').and_capture(
36
+ :remote_host => '127.0.0.1', :timestamp => 20091001075810, :user => nil,
37
+ :http_status => 200, :http_method => 'GET', :http_version => '1.1',
38
+ :duration => 0.0045, :bytes_sent => 1, :remote_logname => nil,
39
+ :path => '/')
40
+ }
41
+
42
+ it { should_not parse_line(irrelevant, 'an irrelevant line') }
43
+ it { should_not parse_line('nonsense', 'a nonsense line') }
44
+ end
4
45
 
5
- context '"Sinatra" access log parsing' do
6
-
7
- before(:each) do
8
- @file_format = RequestLogAnalyzer::FileFormat.load(:rack)
9
- @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
10
- @sample = '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024'
11
- end
12
-
13
- it "should be a valid file format" do
14
- @file_format.should be_valid
15
- end
16
-
17
- it "should parse access lines and capture all of its fields" do
18
- @file_format.should have_line_definition(:access).capturing(:remote_host, :timestamp, :http_method, :path, :http_version,
19
- :http_status, :bytes_sent, :duration)
20
- end
21
-
22
- it "should match the sample line" do
23
- @file_format.parse_line(@sample).should include(:line_definition, :captures)
24
- end
25
-
26
- it "should not match a nonsense line" do
27
- @file_format.parse_line('== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel').should be_nil
28
- end
29
-
30
- it "should parse the sample fields correctly" do
31
- match = @file_format.parse_line(@sample)
32
- match.should_not be_nil
33
- end
34
-
35
- it "should parse and convert the sample fields correctly" do
36
- @log_parser.parse_io(StringIO.new(@sample)) do |request|
37
- request[:remote_host].should == '127.0.0.1'
38
- request[:timestamp].should == 20091123214747
39
- request[:http_method].should == 'GET'
40
- request[:path].should == '/css/stylesheet.css'
41
- request[:http_version].should == '1.1'
42
- request[:http_status].should == 200
43
- request[:bytes_sent].should == 3782
44
- request[:duration].should == 0.0024
45
- end
46
+ describe '#parse_io' do
47
+ let(:snippet) { log_snippet(irrelevant, sample1, sample2, sample3) }
48
+
49
+ it "shouldparse a snippet without warnings" do
50
+ log_parser.should_receive(:handle_request).exactly(3).times
51
+ log_parser.should_not_receive(:warn)
52
+ log_parser.parse_io(snippet)
46
53
  end
47
-
48
54
  end
49
-
50
- end
55
+ end