request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
class DummyInheritedFromRails3 < RequestLogAnalyzer::FileFormat::Rails3
|
|
4
|
+
end
|
|
5
|
+
|
|
6
|
+
describe DummyInheritedFromRails3 do
|
|
7
|
+
|
|
8
|
+
subject { RequestLogAnalyzer::FileFormat.load(DummyInheritedFromRails3) }
|
|
9
|
+
|
|
10
|
+
it { should be_well_formed }
|
|
11
|
+
it { subject.report_trackers.length.should == 11 }
|
|
12
|
+
|
|
13
|
+
end
|
|
@@ -2,66 +2,68 @@ require 'spec_helper'
|
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::LineDefinition do
|
|
4
4
|
|
|
5
|
-
subject
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
5
|
+
subject do
|
|
6
|
+
RequestLogAnalyzer::LineDefinition.new(:test,
|
|
7
|
+
teaser: /Testing /,
|
|
8
|
+
regexp: /Testing (\w+), tries\: (\d+)/,
|
|
9
|
+
captures: [{ name: :what, type: :string }, { name: :tries, type: :integer }]
|
|
10
|
+
)
|
|
11
|
+
end
|
|
11
12
|
|
|
12
13
|
describe '#matches' do
|
|
13
14
|
|
|
14
|
-
it
|
|
15
|
-
subject.matches(
|
|
15
|
+
it 'should return false on an unmatching line' do
|
|
16
|
+
subject.matches('nonmatching').should == false
|
|
16
17
|
end
|
|
17
18
|
|
|
18
|
-
it
|
|
19
|
-
subject.matches(
|
|
19
|
+
it 'should return false when only the teaser matches' do
|
|
20
|
+
subject.matches('Testing LineDefinition').should == false
|
|
20
21
|
end
|
|
21
22
|
|
|
22
|
-
it
|
|
23
|
-
subject.matches(
|
|
23
|
+
it 'should parse a line and capture the expected values' do
|
|
24
|
+
subject.matches('Testing LineDefinition, tries: 123').should == { line_definition: subject, captures: %w(LineDefinition 123) }
|
|
24
25
|
end
|
|
25
26
|
|
|
26
|
-
it
|
|
27
|
-
subject.captures?(:what).should
|
|
28
|
-
subject.captures?(:tries).should
|
|
29
|
-
subject.captures?(:bogus).should
|
|
27
|
+
it 'should know which names it can capture' do
|
|
28
|
+
subject.captures?(:what).should == true
|
|
29
|
+
subject.captures?(:tries).should == true
|
|
30
|
+
subject.captures?(:bogus).should == false
|
|
30
31
|
end
|
|
31
32
|
end
|
|
32
33
|
|
|
33
34
|
describe '#convert_captured_values' do
|
|
34
|
-
let(:request) { double('request', :
|
|
35
|
+
let(:request) { double('request', convert_value: 'foo') }
|
|
35
36
|
|
|
36
|
-
it
|
|
37
|
+
it 'should call convert_value for every captured value' do
|
|
37
38
|
request.should_receive(:convert_value).twice
|
|
38
|
-
subject.convert_captured_values(
|
|
39
|
+
subject.convert_captured_values(%w(test 123), request)
|
|
39
40
|
end
|
|
40
41
|
|
|
41
|
-
it
|
|
42
|
-
subject.convert_captured_values(
|
|
42
|
+
it 'should set the converted values' do
|
|
43
|
+
subject.convert_captured_values(%w(test 123), request).should == { what: 'foo', tries: 'foo' }
|
|
43
44
|
end
|
|
44
45
|
|
|
45
46
|
context 'when using :provides option' do
|
|
46
|
-
|
|
47
|
-
subject
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
|
|
48
|
+
subject do
|
|
49
|
+
RequestLogAnalyzer::LineDefinition.new(:test,
|
|
50
|
+
regexp: /Hash\: (\{.+\})/,
|
|
51
|
+
captures: [{ name: :hash, type: :hash, provides: { bar: :string } }])
|
|
52
|
+
end
|
|
53
|
+
|
|
52
54
|
before do
|
|
53
|
-
request.stub(:convert_value).with("{:bar=>'baz'}", anything).and_return(:
|
|
55
|
+
request.stub(:convert_value).with("{:bar=>'baz'}", anything).and_return(bar: 'baz')
|
|
54
56
|
request.stub(:convert_value).with('baz', anything).and_return('foo')
|
|
55
57
|
end
|
|
56
58
|
|
|
57
|
-
it
|
|
58
|
-
request.should_receive(:convert_value).with("{:bar=>'baz'}", anything).and_return(:
|
|
59
|
-
request.should_receive(:convert_value).with(
|
|
59
|
+
it 'should call Request#convert_value for the initial hash and the value in the hash' do
|
|
60
|
+
request.should_receive(:convert_value).with("{:bar=>'baz'}", anything).and_return(bar: 'baz')
|
|
61
|
+
request.should_receive(:convert_value).with('baz', anything)
|
|
60
62
|
subject.convert_captured_values(["{:bar=>'baz'}"], request)
|
|
61
63
|
end
|
|
62
64
|
|
|
63
|
-
it
|
|
64
|
-
subject.convert_captured_values(["{:bar=>'baz'}"], request).should include(:
|
|
65
|
+
it 'should return the converted hash' do
|
|
66
|
+
subject.convert_captured_values(["{:bar=>'baz'}"], request).should include(bar: 'foo')
|
|
65
67
|
end
|
|
66
68
|
end
|
|
67
69
|
end
|
|
@@ -3,32 +3,34 @@ require 'spec_helper'
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Merb do
|
|
4
4
|
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:merb) }
|
|
6
|
-
|
|
6
|
+
|
|
7
7
|
it { should be_well_formed }
|
|
8
8
|
it { should have_line_definition(:started).capturing(:timestamp) }
|
|
9
9
|
it { should have_line_definition(:params).capturing(:controller, :action, :namespace) }
|
|
10
10
|
it { should have_line_definition(:completed).capturing(:dispatch_time, :before_filters_time, :action_time, :after_filters_time) }
|
|
11
|
-
it { should
|
|
11
|
+
it { should satisfy { |ff| ff.report_trackers.length == 4 } }
|
|
12
12
|
|
|
13
13
|
describe '#parse_line' do
|
|
14
14
|
let(:started_sample) { '~ Started request handling: Fri Aug 29 11:10:23 +0200 2008' }
|
|
15
|
-
let(:prefixed_started_sample) { '~ Aug 31 18:35:24 typekit-web001 merb: ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009' }
|
|
15
|
+
let(:prefixed_started_sample) { '~ Aug 31 18:35:24 typekit-web001 merb: ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009' }
|
|
16
16
|
let(:params_sample) { '~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"delete", "controller"=>"session"}' }
|
|
17
17
|
let(:completed_sample) { '~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}' }
|
|
18
|
-
|
|
19
|
-
it { should parse_line(started_sample, 'without prefix').as(:started).and_capture(:
|
|
20
|
-
it { should parse_line(prefixed_started_sample, 'with prefix').as(:started).and_capture(:
|
|
21
|
-
it { should parse_line(params_sample).as(:params).and_capture(:
|
|
22
|
-
it
|
|
23
|
-
|
|
18
|
+
|
|
19
|
+
it { should parse_line(started_sample, 'without prefix').as(:started).and_capture(timestamp: 20_080_829_111_023) }
|
|
20
|
+
it { should parse_line(prefixed_started_sample, 'with prefix').as(:started).and_capture(timestamp: 20_090_831_183_525) }
|
|
21
|
+
it { should parse_line(params_sample).as(:params).and_capture(controller: 'session', action: 'delete', namespace: nil) }
|
|
22
|
+
it do
|
|
23
|
+
should parse_line(completed_sample).as(:completed).and_capture(dispatch_time: 0.006117,
|
|
24
|
+
before_filters_time: 0.000712, action_time: 0.005833, after_filters_time: 6.1e-05)
|
|
25
|
+
end
|
|
24
26
|
|
|
25
27
|
it { should_not parse_line('~ nonsense', 'a nonsense line') }
|
|
26
28
|
end
|
|
27
29
|
|
|
28
30
|
describe '#parse_io' do
|
|
29
|
-
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
31
|
+
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
30
32
|
|
|
31
|
-
it
|
|
33
|
+
it 'should parse a log fragment correctly without warnings' do
|
|
32
34
|
log_parser.should_receive(:handle_request).exactly(11).times
|
|
33
35
|
log_parser.should_not_receive(:warn)
|
|
34
36
|
log_parser.parse_file(log_fixture(:merb))
|
|
@@ -3,7 +3,7 @@ require 'spec_helper'
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
4
4
|
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:mysql) }
|
|
6
|
-
|
|
6
|
+
|
|
7
7
|
it { should be_well_formed }
|
|
8
8
|
it { should have_line_definition(:time).capturing(:timestamp) }
|
|
9
9
|
it { should have_line_definition(:user_host).capturing(:user, :host, :ip) }
|
|
@@ -11,10 +11,10 @@ describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
|
11
11
|
it { should have_line_definition(:use_database).capturing(:database) }
|
|
12
12
|
it { should have_line_definition(:query_part).capturing(:query_fragment) }
|
|
13
13
|
it { should have_line_definition(:query).capturing(:query) }
|
|
14
|
-
it { should
|
|
15
|
-
|
|
14
|
+
it { should satisfy { |ff| ff.report_trackers.length == 7 } }
|
|
15
|
+
|
|
16
16
|
describe '#parse_line' do
|
|
17
|
-
|
|
17
|
+
|
|
18
18
|
let(:time_sample) { '# Time: 091112 8:13:56' }
|
|
19
19
|
let(:user_host_sample) { '# User@Host: admin[admin] @ db1 [10.0.0.1]' }
|
|
20
20
|
let(:user_host_wo_host_sample) { '# User@Host: admin[admin] @ [10.0.0.1]' }
|
|
@@ -28,15 +28,15 @@ describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
|
28
28
|
let(:set_insertid_sample) { 'SET insert_id=1250651725;' }
|
|
29
29
|
let(:set_timestamp_insertid_sample) { 'SET timestamp=1250651725, insert_id=45674;' }
|
|
30
30
|
|
|
31
|
-
it { should parse_line(time_sample).as(:time).and_capture(:
|
|
32
|
-
it { should parse_line(user_host_sample).as(:user_host).and_capture(:
|
|
33
|
-
it { should parse_line(user_host_wo_host_sample, 'without host').as(:user_host).and_capture(:
|
|
34
|
-
it { should parse_line(user_host_wo_ip_sample, 'without IP').as(:user_host).and_capture(:
|
|
35
|
-
it { should parse_line(float_query_statistics_sample, 'using floats').as(:query_statistics).and_capture(:
|
|
36
|
-
it { should parse_line(int_query_statistics_sample, 'using integers').as(:query_statistics).and_capture(:
|
|
37
|
-
it { should parse_line(partial_query_sample).as(:query_part).and_capture(:
|
|
38
|
-
it { should parse_line(full_query_sample).as(:query).and_capture(:
|
|
39
|
-
it { should parse_line(use_db_sample).as(:use_database).and_capture(:
|
|
31
|
+
it { should parse_line(time_sample).as(:time).and_capture(timestamp: 20_091_112_081_356) }
|
|
32
|
+
it { should parse_line(user_host_sample).as(:user_host).and_capture(user: 'admin', host: 'db1', ip: '10.0.0.1') }
|
|
33
|
+
it { should parse_line(user_host_wo_host_sample, 'without host').as(:user_host).and_capture(user: 'admin', host: '', ip: '10.0.0.1') }
|
|
34
|
+
it { should parse_line(user_host_wo_ip_sample, 'without IP').as(:user_host).and_capture(user: 'root', host: 'localhost', ip: '') }
|
|
35
|
+
it { should parse_line(float_query_statistics_sample, 'using floats').as(:query_statistics).and_capture(query_time: 10.0, lock_time: 0.0, rows_sent: 1_191_307, rows_examined: 1_191_307) }
|
|
36
|
+
it { should parse_line(int_query_statistics_sample, 'using integers').as(:query_statistics).and_capture(query_time: 10.0, lock_time: 0.0, rows_sent: 1_191_307, rows_examined: 1_191_307) }
|
|
37
|
+
it { should parse_line(partial_query_sample).as(:query_part).and_capture(query_fragment: 'AND clients.index > 0') }
|
|
38
|
+
it { should parse_line(full_query_sample).as(:query).and_capture(query: 'SELECT /*!:int SQL_NO_CACHE */ * FROM events') }
|
|
39
|
+
it { should parse_line(use_db_sample).as(:use_database).and_capture(database: 'db') }
|
|
40
40
|
|
|
41
41
|
it { should_not parse_line(set_timestamp_sample) }
|
|
42
42
|
it { should_not parse_line(set_insertid_sample) }
|
|
@@ -46,48 +46,48 @@ describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
|
46
46
|
describe '#parse_io' do
|
|
47
47
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
48
48
|
|
|
49
|
-
it
|
|
49
|
+
it 'should parse a single line query entry correctly' do
|
|
50
50
|
fixture = <<-EOS
|
|
51
51
|
# Time: 091112 18:13:56
|
|
52
52
|
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
53
53
|
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
54
54
|
SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
|
|
55
55
|
EOS
|
|
56
|
-
|
|
56
|
+
|
|
57
57
|
log_parser.parse_string(fixture) do |request|
|
|
58
58
|
request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
|
|
59
59
|
end
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
-
it
|
|
62
|
+
it 'should parse a multiline query entry correctly' do
|
|
63
63
|
fixture = <<-EOS
|
|
64
64
|
# Time: 091112 18:13:56
|
|
65
65
|
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
66
66
|
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
67
|
-
SELECT * FROM `clients` WHERE (1=1
|
|
67
|
+
SELECT * FROM `clients` WHERE (1=1
|
|
68
68
|
AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
|
|
69
69
|
AND clients.index > 0
|
|
70
70
|
) AND (clients.deleted_at IS NULL);
|
|
71
71
|
EOS
|
|
72
|
-
|
|
72
|
+
|
|
73
73
|
log_parser.parse_string(fixture) do |request|
|
|
74
|
-
request[:query].should ==
|
|
74
|
+
request[:query].should == 'SELECT * FROM clients WHERE (:int=:int AND clients.valid_from < :date AND (clients.valid_to IS NULL or clients.valid_to > :date) AND clients.index > :int ) AND (clients.deleted_at IS NULL)'
|
|
75
75
|
end
|
|
76
76
|
end
|
|
77
77
|
|
|
78
|
-
it
|
|
79
|
-
|
|
78
|
+
it 'should parse a request without timestamp correctly, without warnings' do
|
|
79
|
+
fixture = <<-EOS
|
|
80
80
|
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
81
81
|
# Query_time: 10 Lock_time: 0 Rows_sent: 1191307 Rows_examined: 1191307
|
|
82
82
|
SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
|
|
83
83
|
EOS
|
|
84
|
-
|
|
84
|
+
|
|
85
85
|
log_parser.should_receive(:handle_request).once
|
|
86
86
|
log_parser.should_not_receive(:warn)
|
|
87
87
|
log_parser.parse_string(fixture)
|
|
88
88
|
end
|
|
89
89
|
|
|
90
|
-
it
|
|
90
|
+
it 'should parse a query with context information correctly' do
|
|
91
91
|
fixture = <<-EOS
|
|
92
92
|
# Time: 091112 18:13:56
|
|
93
93
|
# User@Host: admin[admin] @ db1 [10.0.0.1]
|
|
@@ -102,7 +102,7 @@ describe RequestLogAnalyzer::FileFormat::Mysql do
|
|
|
102
102
|
end
|
|
103
103
|
end
|
|
104
104
|
|
|
105
|
-
it
|
|
105
|
+
it 'should find 26 completed sloq query entries' do
|
|
106
106
|
log_parser.should_not_receive(:warn)
|
|
107
107
|
log_parser.should_receive(:handle_request).exactly(26).times
|
|
108
108
|
log_parser.parse_file(log_fixture(:mysql_slow_query))
|
|
@@ -3,69 +3,69 @@ require 'spec_helper'
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Oink do
|
|
4
4
|
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:oink) }
|
|
6
|
-
|
|
6
|
+
|
|
7
7
|
it { should have_line_definition(:memory_usage).capturing(:pid, :memory) }
|
|
8
8
|
it { should have_line_definition(:processing).capturing(:pid, :controller, :action, :ip) }
|
|
9
9
|
it { should have_line_definition(:instance_type_counter).capturing(:pid, :instance_counts) }
|
|
10
|
-
it { should
|
|
10
|
+
it { should satisfy { |ff| ff.report_trackers.length == 12 } }
|
|
11
11
|
|
|
12
12
|
describe '#parse_line' do
|
|
13
|
-
let(:memory_usage_sample) { 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783' }
|
|
13
|
+
let(:memory_usage_sample) { 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783' }
|
|
14
14
|
let(:processing_sample) { 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]' }
|
|
15
|
-
let(:instance_type_counter_sample) {
|
|
15
|
+
let(:instance_type_counter_sample) { 'Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1' }
|
|
16
16
|
|
|
17
|
-
it
|
|
18
|
-
subject.should parse_line(memory_usage_sample).as(:memory_usage).and_capture(:
|
|
17
|
+
it 'should parse a :memory_usage line correctly' do
|
|
18
|
+
subject.should parse_line(memory_usage_sample).as(:memory_usage).and_capture(pid: 67_783, memory: 714_052)
|
|
19
19
|
end
|
|
20
|
-
|
|
21
|
-
it
|
|
22
|
-
subject.should parse_line(processing_sample).as(:processing).and_capture(:
|
|
20
|
+
|
|
21
|
+
it 'should parse the PID from a :processing line correctly' do
|
|
22
|
+
subject.should parse_line(processing_sample).as(:processing).and_capture(pid: 67_783, controller: 'PeopleController', action: 'index', timestamp: 20_080_814_211_630, method: 'GET', ip: '1.1.1.1')
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
it
|
|
26
|
-
subject.should parse_line(instance_type_counter_sample).as(:instance_type_counter).and_capture(:
|
|
25
|
+
it 'should parse a :instance_type_counter correctly' do
|
|
26
|
+
subject.should parse_line(instance_type_counter_sample).as(:instance_type_counter).and_capture(pid: 26_364, instance_counts: { 'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1 })
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
|
-
|
|
29
|
+
|
|
30
30
|
describe '#parse_io' do
|
|
31
31
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
32
|
-
|
|
33
|
-
context
|
|
34
|
-
it
|
|
32
|
+
|
|
33
|
+
context 'Rails 2.2 style log' do
|
|
34
|
+
it 'should parse requests' do
|
|
35
35
|
log_parser.should_receive(:handle_request).exactly(4).times
|
|
36
36
|
log_parser.should_not_receive(:warn)
|
|
37
37
|
log_parser.parse_file(log_fixture(:oink_22))
|
|
38
38
|
end
|
|
39
|
-
|
|
40
|
-
it
|
|
39
|
+
|
|
40
|
+
it 'should not record :memory_diff on first request' do
|
|
41
41
|
log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
42
|
-
request[:memory_diff].should
|
|
42
|
+
request[:memory_diff].should.nil? if log_parser.parsed_requests == 1
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
|
-
|
|
46
|
-
it
|
|
45
|
+
|
|
46
|
+
it 'should record :memory_diff of 2nd tracked PID' do
|
|
47
47
|
log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
48
|
-
request[:memory_diff].should ==
|
|
48
|
+
request[:memory_diff].should == 50_000 * 1024 if log_parser.parsed_requests == 3
|
|
49
49
|
end
|
|
50
50
|
end
|
|
51
|
-
|
|
52
|
-
it
|
|
51
|
+
|
|
52
|
+
it 'should record :memory_diff of 1st tracked PID' do
|
|
53
53
|
log_parser.parse_file(log_fixture(:oink_22)) do |request|
|
|
54
|
-
request[:memory_diff].should ==
|
|
54
|
+
request[:memory_diff].should == 30_000 * 1024 if log_parser.parsed_requests == 4
|
|
55
55
|
end
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
|
-
|
|
58
|
+
|
|
59
59
|
context 'Rails 2.2 style log w/failure' do
|
|
60
|
-
it
|
|
60
|
+
it 'should parse requests' do
|
|
61
61
|
log_parser.should_receive(:handle_request).exactly(4).times
|
|
62
62
|
log_parser.should_not_receive(:warn)
|
|
63
63
|
log_parser.parse_file(log_fixture(:oink_22_failure))
|
|
64
64
|
end
|
|
65
|
-
|
|
66
|
-
it
|
|
65
|
+
|
|
66
|
+
it 'should ignore memory changes when a failure occurs' do
|
|
67
67
|
log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
|
|
68
|
-
request[:memory_diff].should
|
|
68
|
+
request[:memory_diff].should.nil? if log_parser.parsed_requests == 4
|
|
69
69
|
end
|
|
70
70
|
end
|
|
71
71
|
end
|
|
@@ -4,23 +4,23 @@ describe RequestLogAnalyzer::FileFormat::Postgresql do
|
|
|
4
4
|
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
|
|
6
6
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
7
|
-
|
|
7
|
+
|
|
8
8
|
it { should be_well_formed }
|
|
9
9
|
|
|
10
10
|
describe '#parse_line' do
|
|
11
|
-
it
|
|
11
|
+
it 'should parse a :query line correctly' do
|
|
12
12
|
line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
|
|
13
|
-
subject.should parse_line(line).as(:query).and_capture(:
|
|
13
|
+
subject.should parse_line(line).as(:query).and_capture(timestamp: 20_101_010_135_207, query_fragment: 'SELECT * FROM "delayed_jobs"')
|
|
14
14
|
end
|
|
15
|
-
|
|
16
|
-
it
|
|
15
|
+
|
|
16
|
+
it 'should parse a :query_fragment line correctly' do
|
|
17
17
|
line = ' ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
|
|
18
|
-
subject.should parse_line(line).as(:query_fragment).and_capture(:
|
|
18
|
+
subject.should parse_line(line).as(:query_fragment).and_capture(query_fragment: '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
it
|
|
21
|
+
it 'should parse a :query line correctly' do
|
|
22
22
|
line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"'
|
|
23
|
-
subject.should parse_line(line).as(:query).and_capture(:
|
|
23
|
+
subject.should parse_line(line).as(:query).and_capture(query_time: 0.710)
|
|
24
24
|
end
|
|
25
25
|
end
|
|
26
26
|
|
|
@@ -31,7 +31,7 @@ describe RequestLogAnalyzer::FileFormat::Postgresql do
|
|
|
31
31
|
# ', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id"
|
|
32
32
|
# 2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081
|
|
33
33
|
# EOS
|
|
34
|
-
#
|
|
34
|
+
#
|
|
35
35
|
# log_parser.should_not_receive(:warn)
|
|
36
36
|
# log_parser.parse_string(fixture) do |request|
|
|
37
37
|
# request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::Rack do
|
|
4
|
-
|
|
5
|
-
subject { RequestLogAnalyzer::FileFormat.load(:rack)}
|
|
4
|
+
|
|
5
|
+
subject { RequestLogAnalyzer::FileFormat.load(:rack) }
|
|
6
6
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
7
|
-
|
|
7
|
+
|
|
8
8
|
it { should be_well_formed }
|
|
9
|
-
it
|
|
10
|
-
|
|
9
|
+
it do
|
|
10
|
+
should have_line_definition(:access).capturing(:remote_host, :user, :remote_logname,
|
|
11
|
+
:timestamp, :http_method, :path, :http_version, :http_status, :bytes_sent, :duration)
|
|
12
|
+
end
|
|
11
13
|
|
|
12
|
-
it { should
|
|
14
|
+
it { should satisfy { |ff| ff.report_trackers.length == 7 } }
|
|
13
15
|
|
|
14
16
|
let(:sample1) { '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024' }
|
|
15
17
|
let(:sample2) { '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453' }
|
|
@@ -17,36 +19,39 @@ describe RequestLogAnalyzer::FileFormat::Rack do
|
|
|
17
19
|
let(:irrelevant) { '== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel' }
|
|
18
20
|
|
|
19
21
|
describe '#parse_line' do
|
|
20
|
-
|
|
21
|
-
it
|
|
22
|
-
|
|
23
|
-
:
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
:
|
|
32
|
-
:
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
:
|
|
40
|
-
|
|
41
|
-
|
|
22
|
+
|
|
23
|
+
it do
|
|
24
|
+
should parse_line(sample1, 'a sample access line').and_capture(
|
|
25
|
+
remote_host: '127.0.0.1', timestamp: 20_091_123_214_747, user: nil,
|
|
26
|
+
http_status: 200, http_method: 'GET', http_version: '1.1',
|
|
27
|
+
duration: 0.0024, bytes_sent: 3782, remote_logname: nil,
|
|
28
|
+
path: '/css/stylesheet.css')
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
it do
|
|
32
|
+
should parse_line(sample2, 'another sample access line').and_capture(
|
|
33
|
+
remote_host: '127.0.0.1', timestamp: 20_090_916_074_008, user: nil,
|
|
34
|
+
http_status: 500, http_method: 'GET', http_version: '1.1',
|
|
35
|
+
duration: 0.0453, bytes_sent: 63_183, remote_logname: nil,
|
|
36
|
+
path: '/favicon.ico')
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
it do
|
|
40
|
+
should parse_line(sample3, 'a third sample access line').and_capture(
|
|
41
|
+
remote_host: '127.0.0.1', timestamp: 20_091_001_075_810, user: nil,
|
|
42
|
+
http_status: 200, http_method: 'GET', http_version: '1.1',
|
|
43
|
+
duration: 0.0045, bytes_sent: 1, remote_logname: nil,
|
|
44
|
+
path: '/')
|
|
45
|
+
end
|
|
46
|
+
|
|
42
47
|
it { should_not parse_line(irrelevant, 'an irrelevant line') }
|
|
43
|
-
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
48
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
44
49
|
end
|
|
45
50
|
|
|
46
51
|
describe '#parse_io' do
|
|
47
52
|
let(:snippet) { log_snippet(irrelevant, sample1, sample2, sample3) }
|
|
48
|
-
|
|
49
|
-
it
|
|
53
|
+
|
|
54
|
+
it 'shouldparse a snippet without warnings' do
|
|
50
55
|
log_parser.should_receive(:handle_request).exactly(3).times
|
|
51
56
|
log_parser.should_not_receive(:warn)
|
|
52
57
|
log_parser.parse_io(snippet)
|