request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -3,34 +3,40 @@ require 'spec_helper'
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
4
4
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:delayed_job21) }
|
6
|
-
|
6
|
+
|
7
7
|
it { should be_well_formed }
|
8
8
|
it { should have_line_definition(:job_lock).capturing(:timestamp, :job, :host, :pid) }
|
9
9
|
it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) }
|
10
|
-
it { should
|
10
|
+
it { should satisfy { |ff| ff.report_trackers.length == 4 } }
|
11
11
|
|
12
12
|
describe '#parse_line' do
|
13
|
-
|
14
|
-
let(:job_lock_sample1) {
|
15
|
-
let(:job_lock_sample2) {
|
13
|
+
|
14
|
+
let(:job_lock_sample1) { '2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' }
|
15
|
+
let(:job_lock_sample2) { '2010-05-17T17:37:34+0000: [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' }
|
16
16
|
let(:job_completed_sample1) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676' }
|
17
17
|
|
18
|
-
it
|
19
|
-
|
18
|
+
it do
|
19
|
+
should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture(
|
20
|
+
timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888)
|
21
|
+
end
|
22
|
+
|
23
|
+
it do
|
24
|
+
should parse_line(job_lock_sample2, 'with multiple workers').as(:job_lock).and_capture(
|
25
|
+
timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888)
|
26
|
+
end
|
20
27
|
|
21
|
-
it
|
22
|
-
|
28
|
+
it do
|
29
|
+
should parse_line(job_completed_sample1).as(:job_completed).and_capture(
|
30
|
+
timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob')
|
31
|
+
end
|
23
32
|
|
24
|
-
it { should parse_line(job_completed_sample1).as(:job_completed).and_capture(
|
25
|
-
:timestamp => 20100517173735, :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888, :job => 'S3FileJob') }
|
26
|
-
|
27
33
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
28
34
|
end
|
29
|
-
|
35
|
+
|
30
36
|
describe '#parse_io' do
|
31
|
-
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
32
|
-
|
33
|
-
it
|
37
|
+
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
38
|
+
|
39
|
+
it 'should parse a batch of completed jobs without warnings' do
|
34
40
|
fragment = log_snippet(<<-EOLOG)
|
35
41
|
2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
|
36
42
|
2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
|
@@ -6,35 +6,41 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob2 do
|
|
6
6
|
|
7
7
|
it { should be_well_formed }
|
8
8
|
it { should have_line_definition(:job_lock).capturing(:timestamp, :job, :host, :pid) }
|
9
|
-
it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) }
|
10
|
-
it { should
|
11
|
-
|
9
|
+
it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) }
|
10
|
+
it { should satisfy { |ff| ff.report_trackers.length == 4 } }
|
11
|
+
|
12
12
|
describe '#parse_line' do
|
13
|
-
|
14
|
-
let(:job_lock_sample1) {
|
15
|
-
let(:job_lock_sample2) {
|
13
|
+
|
14
|
+
let(:job_lock_sample1) { '2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' }
|
15
|
+
let(:job_lock_sample2) { '2010-05-17T17:37:34+0000: * [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' }
|
16
16
|
let(:job_completed_sample) { '2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676' }
|
17
|
-
let(:starting_sample) { '2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888' }
|
17
|
+
let(:starting_sample) { '2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888' }
|
18
18
|
let(:summary_sample) { '3 jobs processed at 0.3163 j/s, 0 failed ...' }
|
19
|
-
|
20
|
-
it { should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture(
|
21
|
-
:timestamp => 20100517173734, :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888) }
|
22
19
|
|
23
|
-
it
|
24
|
-
|
20
|
+
it do
|
21
|
+
should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture(
|
22
|
+
timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888)
|
23
|
+
end
|
25
24
|
|
26
|
-
it
|
27
|
-
|
25
|
+
it do
|
26
|
+
should parse_line(job_lock_sample2, 'with multiple workers').as(:job_lock).and_capture(
|
27
|
+
timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888)
|
28
|
+
end
|
29
|
+
|
30
|
+
it do
|
31
|
+
should parse_line(job_completed_sample).as(:job_completed).and_capture(timestamp: 20_100_517_173_735,
|
32
|
+
duration: 1.0676, host: 'hostname.co.uk', pid: 11_888)
|
33
|
+
end
|
28
34
|
|
29
35
|
it { should_not parse_line(starting_sample, 'a starting line') }
|
30
36
|
it { should_not parse_line(summary_sample, 'a summary line') }
|
31
37
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
32
38
|
end
|
33
|
-
|
39
|
+
|
34
40
|
describe '#parse_io' do
|
35
41
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
36
42
|
|
37
|
-
it
|
43
|
+
it 'should parse a batch of completed jobs without warnings' do
|
38
44
|
fragment = <<-EOLOG
|
39
45
|
2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
|
40
46
|
2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
|
@@ -8,8 +8,7 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
8
8
|
it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid, :job) }
|
9
9
|
it { should have_line_definition(:job_failed).capturing(:timestamp, :host, :pid, :job, :attempts) }
|
10
10
|
it { should have_line_definition(:job_deleted).capturing(:timestamp, :host, :pid, :job, :failures) }
|
11
|
-
it { should
|
12
|
-
|
11
|
+
it { should satisfy { |ff| ff.report_trackers.length == 6 } }
|
13
12
|
|
14
13
|
describe '#parse_line' do
|
15
14
|
|
@@ -17,23 +16,28 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
17
16
|
let(:job_failed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob.create failed with SocketError: getaddrinfo: Name or service not known - 0 failed attempts' }
|
18
17
|
let(:job_deleted_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] PERMANENTLY removing S3FileJob.create because of 25 consecutive failures.' }
|
19
18
|
|
20
|
-
it
|
21
|
-
|
19
|
+
it do
|
20
|
+
should parse_line(job_completed_sample).as(:job_completed).and_capture(
|
21
|
+
timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create')
|
22
|
+
end
|
22
23
|
|
23
|
-
it
|
24
|
-
|
24
|
+
it do
|
25
|
+
should parse_line(job_failed_sample).as(:job_failed).and_capture(
|
26
|
+
timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create failed with SocketError: getaddrinfo: Name or service not known', attempts: 0)
|
27
|
+
end
|
25
28
|
|
26
|
-
it
|
27
|
-
|
29
|
+
it do
|
30
|
+
should parse_line(job_deleted_sample).as(:job_deleted).and_capture(
|
31
|
+
timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create', failures: 25)
|
32
|
+
end
|
28
33
|
|
29
34
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
30
35
|
end
|
31
36
|
|
32
|
-
|
33
37
|
describe '#parse_io' do
|
34
38
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
35
39
|
|
36
|
-
it
|
40
|
+
it 'should parse a batch of completed jobs without warnings' do
|
37
41
|
fragment = log_snippet(<<-EOLOG)
|
38
42
|
2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
|
39
43
|
2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676
|
@@ -8,8 +8,7 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
8
8
|
it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid, :job) }
|
9
9
|
it { should have_line_definition(:job_failed).capturing(:timestamp, :host, :pid, :job, :attempts, :error) }
|
10
10
|
it { should have_line_definition(:job_deleted).capturing(:timestamp, :host, :pid, :job, :failures) }
|
11
|
-
it { should
|
12
|
-
|
11
|
+
it { should satisfy { |ff| ff.report_trackers.length == 6 } }
|
13
12
|
|
14
13
|
describe '#parse_line' do
|
15
14
|
|
@@ -17,23 +16,28 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
17
16
|
let(:job_failed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534785) FAILED (0 prior attempts) with SocketError: getaddrinfo: Name or service not known' }
|
18
17
|
let(:job_deleted_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534785) REMOVED permanently because of 25 consecutive failures' }
|
19
18
|
|
20
|
-
it
|
21
|
-
|
19
|
+
it do
|
20
|
+
should parse_line(job_completed_sample).as(:job_completed).and_capture(
|
21
|
+
timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create')
|
22
|
+
end
|
22
23
|
|
23
|
-
it
|
24
|
-
|
24
|
+
it do
|
25
|
+
should parse_line(job_failed_sample).as(:job_failed).and_capture(
|
26
|
+
timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create (id=534785)', attempts: 0, error: 'SocketError: getaddrinfo: Name or service not known')
|
27
|
+
end
|
25
28
|
|
26
|
-
it
|
27
|
-
|
29
|
+
it do
|
30
|
+
should parse_line(job_deleted_sample).as(:job_deleted).and_capture(
|
31
|
+
timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create (id=534785)', failures: 25)
|
32
|
+
end
|
28
33
|
|
29
34
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
30
35
|
end
|
31
36
|
|
32
|
-
|
33
37
|
describe '#parse_io' do
|
34
38
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
35
39
|
|
36
|
-
it
|
40
|
+
it 'should parse a batch of completed jobs without warnings' do
|
37
41
|
fragment = log_snippet(<<-EOLOG)
|
38
42
|
2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
|
39
43
|
2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob (id=534785) COMPLETED after 1.0676
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
4
|
-
|
4
|
+
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:delayed_job) }
|
6
6
|
|
7
7
|
it { should be_well_formed }
|
@@ -9,28 +9,28 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
9
9
|
it { should have_line_definition(:job_completed).capturing(:completed_job, :duration) }
|
10
10
|
it { should have_line_definition(:job_lock_failed).capturing(:locked_job) }
|
11
11
|
it { should have_line_definition(:job_failed).capturing(:failed_job, :attempts, :exception) }
|
12
|
-
it { should
|
12
|
+
it { should satisfy { |ff| ff.report_trackers.length == 3 } }
|
13
13
|
|
14
14
|
describe '#parse_line' do
|
15
15
|
let(:job_lock_sample) { '* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver' }
|
16
16
|
let(:job_completed_sample) { '* [JOB] BackgroundJob::ThumbnailSaver completed after 0.7932' }
|
17
17
|
let(:job_lock_failed_sample) { '* [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver' }
|
18
|
-
let(:job_failed_sample) { "* [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 1 failed attempts" }
|
18
|
+
let(:job_failed_sample) { "* [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 1 failed attempts" }
|
19
19
|
let(:summary_sample) { '1 jobs processed at 1.0834 j/s, 0 failed ...' }
|
20
20
|
|
21
|
-
it { should parse_line(job_lock_sample).as(:job_lock).and_capture(:
|
22
|
-
it { should parse_line(job_completed_sample).as(:job_completed).and_capture(:
|
23
|
-
it { should parse_line(job_lock_failed_sample).as(:job_lock_failed).and_capture(:
|
24
|
-
it { should parse_line(job_failed_sample).as(:job_failed).and_capture(:
|
21
|
+
it { should parse_line(job_lock_sample).as(:job_lock).and_capture(job: 'BackgroundJob::ThumbnailSaver') }
|
22
|
+
it { should parse_line(job_completed_sample).as(:job_completed).and_capture(duration: 0.7932, completed_job: 'BackgroundJob::ThumbnailSaver') }
|
23
|
+
it { should parse_line(job_lock_failed_sample).as(:job_lock_failed).and_capture(locked_job: 'BackgroundJob::ThumbnailSaver') }
|
24
|
+
it { should parse_line(job_failed_sample).as(:job_failed).and_capture(attempts: 1, failed_job: 'BackgroundJob::ThumbnailSaver', exception: 'ActiveRecord::RecordNotFound') }
|
25
25
|
|
26
26
|
it { should_not parse_line(summary_sample, 'a summary line') }
|
27
27
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
28
28
|
end
|
29
29
|
|
30
30
|
describe '#parse_io' do
|
31
|
-
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
32
|
-
|
33
|
-
it
|
31
|
+
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
32
|
+
|
33
|
+
it 'should parse a batch of completed jobs without warnings' do
|
34
34
|
fragment = log_snippet(<<-EOLOG)
|
35
35
|
* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
|
36
36
|
* [JOB] BackgroundJob::ThumbnailSaver completed after 0.9114
|
@@ -43,8 +43,8 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
|
|
43
43
|
log_parser.should_not_receive(:warn)
|
44
44
|
log_parser.parse_io(fragment)
|
45
45
|
end
|
46
|
-
|
47
|
-
it
|
46
|
+
|
47
|
+
it 'should parse a batch with a failed job without warnings' do
|
48
48
|
fragment = log_snippet(<<-EOLOG)
|
49
49
|
* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
|
50
50
|
* [JOB] BackgroundJob::ThumbnailSaver completed after 1.0627
|
@@ -2,27 +2,27 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat do
|
4
4
|
|
5
|
-
describe
|
5
|
+
describe '.format_definition' do
|
6
6
|
|
7
7
|
before(:each) do
|
8
8
|
@first_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base)
|
9
9
|
@second_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base)
|
10
10
|
end
|
11
11
|
|
12
|
-
it
|
13
|
-
@first_file_format.format_definition.direct_test :
|
12
|
+
it 'should specify line definitions directly within the file_format' do
|
13
|
+
@first_file_format.format_definition.direct_test regexp: /test/
|
14
14
|
@first_file_format.should have_line_definition(:direct_test)
|
15
15
|
end
|
16
16
|
|
17
|
-
it
|
17
|
+
it 'specify lines with a block for the format definition' do
|
18
18
|
@first_file_format.format_definition do |format|
|
19
|
-
format.block_test :
|
19
|
+
format.block_test regexp: /test (\w+)/, captures: [{ name: :tester, type: :string }]
|
20
20
|
end
|
21
21
|
|
22
22
|
@first_file_format.should have_line_definition(:block_test).capturing(:tester)
|
23
23
|
end
|
24
24
|
|
25
|
-
it
|
25
|
+
it 'should specify a line with a block' do
|
26
26
|
@first_file_format.format_definition.hash_test do |line|
|
27
27
|
line.regexp = /test/
|
28
28
|
line.captures = []
|
@@ -31,39 +31,39 @@ describe RequestLogAnalyzer::FileFormat do
|
|
31
31
|
@first_file_format.should have_line_definition(:hash_test)
|
32
32
|
end
|
33
33
|
|
34
|
-
it
|
35
|
-
@first_file_format.format_definition.first
|
36
|
-
@second_file_format.format_definition.second :
|
34
|
+
it 'should define lines only for its own language' do
|
35
|
+
@first_file_format.format_definition.first regexp: /test 123/
|
36
|
+
@second_file_format.format_definition.second regexp: /test 456/
|
37
37
|
|
38
|
-
@first_file_format.should
|
39
|
-
@first_file_format.should_not
|
38
|
+
@first_file_format.should have_line_definition(:first)
|
39
|
+
@first_file_format.should_not have_line_definition(:second)
|
40
40
|
@second_file_format.should_not have_line_definition(:first)
|
41
|
-
@second_file_format.should
|
41
|
+
@second_file_format.should have_line_definition(:second)
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
45
|
-
describe
|
45
|
+
describe '.load' do
|
46
46
|
|
47
|
-
it
|
47
|
+
it 'should return an instance of a FileFormat class' do
|
48
48
|
@file_format = RequestLogAnalyzer::FileFormat.load(TestingFormat)
|
49
49
|
@file_format.should be_kind_of(TestingFormat)
|
50
50
|
end
|
51
51
|
|
52
|
-
it
|
52
|
+
it 'should return itself if it already is a FileFormat::Base instance' do
|
53
53
|
@file_format = RequestLogAnalyzer::FileFormat.load(testing_format)
|
54
54
|
@file_format.should be_kind_of(TestingFormat)
|
55
55
|
end
|
56
56
|
|
57
|
-
it
|
57
|
+
it 'should load a predefined file format from the /file_format dir' do
|
58
58
|
@file_format = RequestLogAnalyzer::FileFormat.load(:rails)
|
59
59
|
@file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
|
60
60
|
end
|
61
61
|
|
62
|
-
it
|
62
|
+
it 'should load a provided format file' do
|
63
63
|
format_filename = File.expand_path('../../lib/testing_format.rb', File.dirname(__FILE__))
|
64
64
|
@file_format = RequestLogAnalyzer::FileFormat.load(format_filename)
|
65
65
|
@file_format.should be_kind_of(TestingFormat)
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
end
|
69
|
-
end
|
69
|
+
end
|
@@ -3,37 +3,37 @@ require 'spec_helper'
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat do
|
4
4
|
|
5
5
|
describe '.autodetect' do
|
6
|
-
it
|
6
|
+
it 'should autodetect a Merb log' do
|
7
7
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:merb))
|
8
8
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Merb)
|
9
9
|
end
|
10
10
|
|
11
|
-
it
|
11
|
+
it 'should autodetect a MySQL slow query log' do
|
12
12
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:mysql_slow_query))
|
13
13
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Mysql)
|
14
14
|
end
|
15
15
|
|
16
|
-
it
|
16
|
+
it 'should autodetect a Rails 1.x log' do
|
17
17
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_1x))
|
18
18
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rails)
|
19
19
|
end
|
20
20
|
|
21
|
-
it
|
21
|
+
it 'should autodetect a Rails 2.x log' do
|
22
22
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_22))
|
23
23
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::RailsDevelopment)
|
24
24
|
end
|
25
25
|
|
26
|
-
it
|
26
|
+
it 'should autodetect an Apache access log' do
|
27
27
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:apache_common))
|
28
28
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Apache)
|
29
29
|
end
|
30
30
|
|
31
|
-
it
|
31
|
+
it 'should autodetect a Rack access log' do
|
32
32
|
file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:sinatra))
|
33
33
|
file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rack)
|
34
34
|
end
|
35
35
|
|
36
|
-
it
|
36
|
+
it 'should not find any file format with a bogus file' do
|
37
37
|
RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:test_order)).should be_nil
|
38
38
|
end
|
39
39
|
end
|
@@ -11,7 +11,7 @@ describe RequestLogAnalyzer::FileFormat::Haproxy do
|
|
11
11
|
it { should have_line_definition(:haproxy12).capturing(:client_ip, :timestamp, :frontend_name, :server_name, :tq, :tw, :tc, :tr, :tt, :status_code, :bytes_read, :captured_request_cookie, :captured_response_cookie, :termination_event_code, :terminated_session_state, :clientside_persistence_cookie, :serverside_persistence_cookie, :srv_conn, :listener_conn, :process_conn, :srv_queue, :backend_queue, :captured_request_headers, :captured_response_headers, :http_request) }
|
12
12
|
it { should have_line_definition(:haproxy11).capturing(:client_ip, :timestamp, :frontend_name, :server_name, :tq, :tc, :tr, :tt, :status_code, :bytes_read, :captured_request_cookie, :captured_response_cookie, :termination_event_code, :terminated_session_state, :clientside_persistence_cookie, :serverside_persistence_cookie, :listener_conn, :process_conn, :captured_request_headers, :captured_response_headers, :http_request) }
|
13
13
|
|
14
|
-
it { should
|
14
|
+
it { should satisfy { |ff| ff.report_trackers.length == 14 } }
|
15
15
|
|
16
16
|
let(:sample_haproxy13) { 'Feb 6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1"' }
|
17
17
|
let(:sample_haproxy12) { 'Mar 15 06:36:49 localhost haproxy[9367]: 127.0.0.1:38990 [15/Mar/2011:06:36:45.103] as-proxy mc-search-2 0/0/0/730/731 200 29404 - - --NN 2/54/54 0/0 {66.249.68.216} {} "GET /neighbor/26014153 HTTP/1.0" ' }
|
@@ -19,61 +19,65 @@ describe RequestLogAnalyzer::FileFormat::Haproxy do
|
|
19
19
|
let(:sample_errors) { 'haproxy[18113]: 127.0.0.1:34549 [15/Oct/2003:15:19:06.103] px-http px-http/<NOSRV> -1/-1/-1/-1/+50001 408 +2750 - - cR-- 2/2/2/0/+2 0/0 ""' }
|
20
20
|
|
21
21
|
describe '#parse_line' do
|
22
|
-
it
|
23
|
-
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
29
|
-
:
|
30
|
-
:
|
31
|
-
:
|
32
|
-
:
|
33
|
-
|
22
|
+
it do
|
23
|
+
should parse_line(sample_haproxy13, 'an haproxy 1.3 access line').and_capture(
|
24
|
+
client_ip: '10.0.1.2', tq: 0.010, captured_request_cookie: nil,
|
25
|
+
timestamp: 20_090_206_121_414, tw: 0.000, captured_response_cookie: nil,
|
26
|
+
frontend_name: 'http-in', tc: 0.030, clientside_persistence_cookie: nil,
|
27
|
+
backend_name: 'static', tr: 0.069, serverside_persistence_cookie: nil,
|
28
|
+
server_name: 'srv1', tt: 0.109, termination_event_code: nil,
|
29
|
+
status_code: 200, actconn: 1, terminated_session_state: nil,
|
30
|
+
bytes_read: 2750, feconn: 1, captured_request_headers: '{1wt.eu}',
|
31
|
+
backend_queue: 0, beconn: 1, captured_response_headers: nil,
|
32
|
+
retries: 0, srv_conn: 1, srv_queue: 0,
|
33
|
+
http_request: 'GET /index.html HTTP/1.1')
|
34
|
+
end
|
35
|
+
|
36
|
+
it do
|
37
|
+
should parse_line(sample_haproxy12, 'an haproxy 1.2 access line').and_capture(
|
38
|
+
client_ip: '127.0.0.1', tq: 0.000, captured_request_cookie: nil,
|
39
|
+
timestamp: 20_110_315_063_645, tw: 0.000, captured_response_cookie: nil,
|
40
|
+
frontend_name: 'as-proxy', tc: 0.000, clientside_persistence_cookie: 'N',
|
41
|
+
server_name: 'mc-search-2', tr: 0.730, serverside_persistence_cookie: 'N',
|
42
|
+
status_code: 200, tt: 0.731, termination_event_code: nil,
|
43
|
+
bytes_read: 29_404, listener_conn: 54, terminated_session_state: nil,
|
44
|
+
backend_queue: 0, process_conn: 54, captured_request_headers: '{66.249.68.216}',
|
45
|
+
srv_queue: 0, srv_conn: 2, captured_response_headers: nil,
|
46
|
+
http_request: 'GET /neighbor/26014153 HTTP/1.0')
|
47
|
+
end
|
48
|
+
|
49
|
+
it do
|
50
|
+
should parse_line(sample_haproxy11, 'an haproxy 1.1 access line').and_capture(
|
51
|
+
client_ip: '127.0.0.1', tq: 0.009, captured_request_cookie: nil,
|
52
|
+
timestamp: 20_031_015_083_217, tc: 0.007, captured_response_cookie: nil,
|
53
|
+
frontend_name: 'relais-http', tr: 0.014, clientside_persistence_cookie: nil,
|
54
|
+
server_name: 'Srv1', tt: 0.030, serverside_persistence_cookie: nil,
|
55
|
+
status_code: 502, listener_conn: 2, termination_event_code: 'P',
|
56
|
+
bytes_read: 243, process_conn: 3, terminated_session_state: 'H',
|
57
|
+
captured_request_headers: nil, captured_response_headers: nil,
|
58
|
+
http_request: nil)
|
59
|
+
end
|
34
60
|
|
35
|
-
it
|
36
|
-
|
37
|
-
:
|
38
|
-
:
|
39
|
-
:
|
40
|
-
:
|
41
|
-
:
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
61
|
+
it do
|
62
|
+
should parse_line(sample_errors, 'a failed access line').and_capture(
|
63
|
+
timestamp: 20_031_015_151_906, tq: nil, captured_request_cookie: nil,
|
64
|
+
server_name: '<NOSRV>', tw: nil, captured_response_cookie: nil,
|
65
|
+
bytes_read: 2750, tc: nil, clientside_persistence_cookie: nil,
|
66
|
+
retries: 2, tr: nil, serverside_persistence_cookie: nil,
|
67
|
+
http_request: nil, tt: 50.001, termination_event_code: 'c',
|
68
|
+
terminated_session_state: 'R',
|
69
|
+
captured_request_headers: nil,
|
70
|
+
captured_response_headers: nil)
|
71
|
+
end
|
46
72
|
|
47
|
-
it { should parse_line(sample_haproxy11, 'an haproxy 1.1 access line').and_capture(
|
48
|
-
:client_ip => '127.0.0.1', :tq => 0.009, :captured_request_cookie => nil,
|
49
|
-
:timestamp => 20031015083217, :tc => 0.007, :captured_response_cookie => nil,
|
50
|
-
:frontend_name => 'relais-http',:tr => 0.014, :clientside_persistence_cookie => nil,
|
51
|
-
:server_name => 'Srv1', :tt => 0.030, :serverside_persistence_cookie => nil,
|
52
|
-
:status_code => 502, :listener_conn => 2,:termination_event_code => 'P',
|
53
|
-
:bytes_read => 243, :process_conn => 3, :terminated_session_state => 'H',
|
54
|
-
:captured_request_headers => nil, :captured_response_headers => nil,
|
55
|
-
:http_request => nil)
|
56
|
-
}
|
57
|
-
|
58
|
-
it { should parse_line(sample_errors, 'a failed access line').and_capture(
|
59
|
-
:timestamp => 20031015151906, :tq => nil, :captured_request_cookie => nil,
|
60
|
-
:server_name => '<NOSRV>', :tw => nil, :captured_response_cookie => nil,
|
61
|
-
:bytes_read => 2750, :tc => nil, :clientside_persistence_cookie => nil,
|
62
|
-
:retries => 2, :tr => nil, :serverside_persistence_cookie => nil,
|
63
|
-
:http_request => nil, :tt => 50.001, :termination_event_code => 'c',
|
64
|
-
:terminated_session_state => 'R',
|
65
|
-
:captured_request_headers => nil,
|
66
|
-
:captured_response_headers => nil)
|
67
|
-
}
|
68
|
-
|
69
73
|
it { should_not parse_line('nonsense') }
|
70
74
|
end
|
71
|
-
|
75
|
+
|
72
76
|
describe '#parse_io' do
|
73
77
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
74
78
|
let(:snippet) { log_snippet(sample_haproxy13, sample_haproxy12, sample_haproxy11, sample_errors, 'nonsense') }
|
75
|
-
|
76
|
-
it
|
79
|
+
|
80
|
+
it 'should parse a log snippet without warnings' do
|
77
81
|
log_parser.should_receive(:handle_request).exactly(4).times
|
78
82
|
log_parser.should_not_receive(:warn)
|
79
83
|
log_parser.parse_io(snippet)
|