request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -16,7 +16,7 @@ describe RequestLogAnalyzer::Database do
|
|
16
16
|
# FileFormat-agnostic classes
|
17
17
|
default_orm_class_names.each do |const|
|
18
18
|
it "should create the default #{const} constant" do
|
19
|
-
RequestLogAnalyzer::Database.const_defined?(const).should
|
19
|
+
RequestLogAnalyzer::Database.const_defined?(const).should == true
|
20
20
|
end
|
21
21
|
|
22
22
|
it "should create the default #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
|
@@ -25,9 +25,9 @@ describe RequestLogAnalyzer::Database do
|
|
25
25
|
end
|
26
26
|
|
27
27
|
# Some Fileformat-specific classes
|
28
|
-
|
28
|
+
%w(CompletedLine ProcessingLine).each do |const|
|
29
29
|
it "should create the #{const} constant" do
|
30
|
-
Object.const_defined?(const).should
|
30
|
+
Object.const_defined?(const).should == true
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should create the #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
|
@@ -73,11 +73,11 @@ describe RequestLogAnalyzer::Database do
|
|
73
73
|
end
|
74
74
|
end
|
75
75
|
|
76
|
-
testing_format.line_definitions.each do |name,
|
76
|
+
testing_format.line_definitions.each do |name, _definition|
|
77
77
|
|
78
78
|
it "should create the #{(name.to_s + '_line').camelize} class for #{name.inspect} lines" do
|
79
79
|
@database.send :create_database_schema!
|
80
|
-
Object.const_defined?("#{name}_line".camelize).should
|
80
|
+
Object.const_defined?("#{name}_line".camelize).should == true
|
81
81
|
end
|
82
82
|
|
83
83
|
it "should create the #{name.to_s + '_lines'} table for the parsed #{name.inspect} lines" do
|
@@ -106,27 +106,27 @@ describe RequestLogAnalyzer::Database do
|
|
106
106
|
|
107
107
|
after(:each) { @database.remove_orm_classes! }
|
108
108
|
|
109
|
-
it
|
109
|
+
it 'should call :subclass_from_table when a table name is given as string' do
|
110
110
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
|
111
111
|
@database.load_activerecord_class('test_lines')
|
112
112
|
end
|
113
113
|
|
114
|
-
it
|
114
|
+
it 'should call :subclass_from_table when a table name is given as symbol' do
|
115
115
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
|
116
116
|
@database.load_activerecord_class(:test_lines)
|
117
117
|
end
|
118
118
|
|
119
|
-
it
|
119
|
+
it 'should call :subclass_from_table when a LineDefinition is given' do
|
120
120
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_line_definition).and_return(@mock_class)
|
121
121
|
@database.load_activerecord_class(RequestLogAnalyzer::LineDefinition.new(:test))
|
122
122
|
end
|
123
123
|
|
124
|
-
it
|
124
|
+
it 'should define the class in the ORM module' do
|
125
125
|
@database.load_activerecord_class(:test_lines)
|
126
|
-
Object.const_defined?('TestLine').should
|
126
|
+
Object.const_defined?('TestLine').should == true
|
127
127
|
end
|
128
128
|
|
129
|
-
it
|
129
|
+
it 'should add the class to the line_classes array of the database' do
|
130
130
|
@database.load_activerecord_class(:test_lines)
|
131
131
|
@database.line_classes.should include(TestLine)
|
132
132
|
end
|
@@ -5,76 +5,80 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
|
6
6
|
|
7
7
|
it { should be_well_formed }
|
8
|
-
it
|
9
|
-
|
8
|
+
it do
|
9
|
+
should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
|
10
|
+
:key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent)
|
11
|
+
end
|
10
12
|
|
11
|
-
it { should
|
13
|
+
it { should satisfy { |ff| ff.report_trackers.length == 7 } }
|
12
14
|
|
13
15
|
let(:sample_get) { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
|
14
16
|
let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
|
15
17
|
let(:sample_2013) { 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac testbucket [17/Sep/2013:15:53:53 +0000] 87.193.165.87 - A9774F82F053FACE REST.GET.OBJECT somefile.txt "GET /testbucket/somefile.txt HTTP/1.1" 304 - - 1071 9 - "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36" -' }
|
16
18
|
|
17
19
|
describe '#parse_line' do
|
18
|
-
it
|
20
|
+
it do
|
19
21
|
should parse_line(sample_get, 'a GET line').and_capture(
|
20
|
-
:
|
21
|
-
:
|
22
|
-
:
|
23
|
-
:
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
28
|
-
:
|
29
|
-
:
|
30
|
-
:
|
31
|
-
:
|
32
|
-
:
|
33
|
-
:
|
34
|
-
:
|
35
|
-
:
|
36
|
-
:
|
37
|
-
|
22
|
+
bucket_owner: '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
|
23
|
+
bucket: 'sample-bucket',
|
24
|
+
timestamp: 20_061_006_014_214,
|
25
|
+
remote_ip: '207.171.172.6',
|
26
|
+
key: 'object.png',
|
27
|
+
operation: 'REST.GET.OBJECT',
|
28
|
+
requester: '65a011a29cdf8ec533ec3d1ccaae921c',
|
29
|
+
request_id: 'C980091AD89C936A',
|
30
|
+
request_uri: 'GET /sample-bucket/object.png HTTP/1.1',
|
31
|
+
error_code: nil,
|
32
|
+
http_status: 200,
|
33
|
+
total_time: 0.988,
|
34
|
+
turnaround_time: 0.987,
|
35
|
+
bytes_sent: 1243,
|
36
|
+
object_size: 1243,
|
37
|
+
user_agent: 'aranhabot',
|
38
|
+
referer: nil)
|
39
|
+
end
|
38
40
|
|
39
|
-
it
|
40
|
-
|
41
|
-
:
|
42
|
-
:
|
43
|
-
:
|
44
|
-
:
|
45
|
-
:
|
46
|
-
:
|
47
|
-
:
|
48
|
-
:
|
49
|
-
:
|
50
|
-
:
|
51
|
-
:
|
52
|
-
:
|
53
|
-
:
|
54
|
-
:
|
55
|
-
:
|
56
|
-
:
|
57
|
-
|
41
|
+
it do
|
42
|
+
should parse_line(sample_2013, '2013 sample').and_capture(
|
43
|
+
bucket_owner: 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac',
|
44
|
+
bucket: 'testbucket',
|
45
|
+
timestamp: 20_130_917_155_353,
|
46
|
+
remote_ip: '87.193.165.87',
|
47
|
+
requester: '-',
|
48
|
+
request_id: 'A9774F82F053FACE',
|
49
|
+
operation: 'REST.GET.OBJECT',
|
50
|
+
key: 'somefile.txt',
|
51
|
+
request_uri: 'GET /testbucket/somefile.txt HTTP/1.1',
|
52
|
+
http_status: 304,
|
53
|
+
error_code: nil,
|
54
|
+
bytes_sent: 0,
|
55
|
+
object_size: 1071,
|
56
|
+
total_time: 0.009,
|
57
|
+
turnaround_time: 0,
|
58
|
+
referer: 'https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US',
|
59
|
+
user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36')
|
60
|
+
end
|
58
61
|
|
59
|
-
it
|
60
|
-
|
61
|
-
:
|
62
|
-
:
|
63
|
-
:
|
64
|
-
:
|
65
|
-
:
|
66
|
-
:
|
67
|
-
:
|
68
|
-
:
|
69
|
-
:
|
70
|
-
:
|
71
|
-
:
|
72
|
-
:
|
73
|
-
:
|
74
|
-
:
|
75
|
-
:
|
76
|
-
:
|
77
|
-
|
62
|
+
it do
|
63
|
+
should parse_line(sample_copy, 'a COPY line').and_capture(
|
64
|
+
bucket_owner: '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
65
|
+
bucket: 'testbucket',
|
66
|
+
timestamp: 20_100_303_230_459,
|
67
|
+
remote_ip: '174.119.31.76',
|
68
|
+
key: 'files/image.png',
|
69
|
+
operation: 'REST.COPY.OBJECT',
|
70
|
+
requester: '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
71
|
+
request_id: 'ACCC34B843C87BC9',
|
72
|
+
request_uri: 'PUT /files/image.png HTTP/1.1',
|
73
|
+
error_code: nil,
|
74
|
+
http_status: 200,
|
75
|
+
total_time: 0.365,
|
76
|
+
turnaround_time: 0.319,
|
77
|
+
bytes_sent: 234,
|
78
|
+
object_size: 65_957,
|
79
|
+
user_agent: '',
|
80
|
+
referer: nil)
|
81
|
+
end
|
78
82
|
|
79
83
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
80
84
|
end
|
@@ -83,10 +87,10 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
83
87
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
84
88
|
let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
|
85
89
|
|
86
|
-
it
|
90
|
+
it 'should parse requests correctly and not generate warnings' do
|
87
91
|
log_parser.should_receive(:handle_request).twice
|
88
92
|
log_parser.should_not_receive(:warn)
|
89
93
|
log_parser.parse_io(snippet)
|
90
94
|
end
|
91
95
|
end
|
92
|
-
end
|
96
|
+
end
|
@@ -6,41 +6,41 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
6
6
|
let(:format_string) { '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T' }
|
7
7
|
let(:line_definition) { RequestLogAnalyzer::FileFormat::Apache.access_line_definition(format_string) }
|
8
8
|
|
9
|
-
it
|
9
|
+
it 'should create a Regexp to match the line' do
|
10
10
|
line_definition.regexp.should be_kind_of(Regexp)
|
11
11
|
end
|
12
12
|
|
13
|
-
it
|
14
|
-
line_definition.captures.should
|
13
|
+
it 'should create a list of captures for the values in the lines' do
|
14
|
+
line_definition.captures.length.should == 12
|
15
15
|
end
|
16
16
|
|
17
|
-
it
|
17
|
+
it 'should make it a header line' do
|
18
18
|
line_definition.should be_header
|
19
19
|
end
|
20
20
|
|
21
|
-
it
|
21
|
+
it 'should make it a footer line' do
|
22
22
|
line_definition.should be_footer
|
23
23
|
end
|
24
24
|
|
25
|
-
it
|
26
|
-
line_definition.captures?(:duration).should
|
25
|
+
it 'should capture :duration' do
|
26
|
+
line_definition.captures?(:duration).should == true
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
30
|
describe '.access_line_definition' do
|
31
|
-
it
|
31
|
+
it 'should parse values in microseconds when no argument is given to %D' do
|
32
32
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
|
33
|
-
format.should parse_line('12345').and_capture(:
|
33
|
+
format.should parse_line('12345').and_capture(duration: 0.012345)
|
34
34
|
end
|
35
35
|
|
36
|
-
it
|
36
|
+
it 'should parse values in microseconds when micro is given as argument to %D' do
|
37
37
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
|
38
|
-
format.should parse_line('12345').and_capture(:
|
38
|
+
format.should parse_line('12345').and_capture(duration: 0.012345)
|
39
39
|
end
|
40
40
|
|
41
|
-
it
|
41
|
+
it 'should parse values in microseconds when micro is given as argument to %D' do
|
42
42
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
|
43
|
-
format.should parse_line('12345').and_capture(:
|
43
|
+
format.should parse_line('12345').and_capture(duration: 12.345)
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -50,20 +50,21 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
50
50
|
|
51
51
|
it { should be_well_formed }
|
52
52
|
it { should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status) }
|
53
|
-
it { should
|
53
|
+
it { should satisfy { |ff| ff.report_trackers.length == 8 } }
|
54
54
|
end
|
55
55
|
|
56
56
|
context '"vhost_combined" access log parsing' do
|
57
57
|
subject { RequestLogAnalyzer::FileFormat.load(:apache, :vhost_combined) }
|
58
58
|
describe '#parse_line' do
|
59
|
-
let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683'}
|
60
|
-
|
61
|
-
it
|
62
|
-
|
63
|
-
:
|
64
|
-
:
|
65
|
-
:
|
66
|
-
|
59
|
+
let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683' }
|
60
|
+
|
61
|
+
it do
|
62
|
+
should parse_line(sample1, 'a sample line').and_capture(
|
63
|
+
remote_host: '72.204.80.86', remote_logname: nil, user: nil,
|
64
|
+
vhost: 'psi-equipment.od1.vtiger.com', duration: 1.0,
|
65
|
+
timestamp: 20_131_019_154_154, http_status: 304, http_method: 'GET',
|
66
|
+
http_version: '1.1', bytes_sent: 0)
|
67
|
+
end
|
67
68
|
end
|
68
69
|
end
|
69
70
|
|
@@ -72,32 +73,34 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
72
73
|
|
73
74
|
it { should be_well_formed }
|
74
75
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent) }
|
75
|
-
it { should
|
76
|
+
it { should satisfy { |ff| ff.report_trackers.length == 6 } }
|
76
77
|
|
77
78
|
describe '#parse_line' do
|
78
79
|
|
79
80
|
let(:sample1) { '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223' }
|
80
81
|
let(:sample2) { '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414' }
|
81
82
|
|
82
|
-
it
|
83
|
-
|
84
|
-
:
|
85
|
-
:
|
86
|
-
|
83
|
+
it do
|
84
|
+
should parse_line(sample1, 'a sample line').and_capture(
|
85
|
+
remote_host: '1.129.119.13', remote_logname: nil, user: nil,
|
86
|
+
timestamp: 20_090_908_075_409, http_status: 200, http_method: 'GET',
|
87
|
+
http_version: '1.0', bytes_sent: 8223)
|
88
|
+
end
|
87
89
|
|
88
|
-
it
|
89
|
-
|
90
|
-
:
|
91
|
-
:
|
92
|
-
|
90
|
+
it do
|
91
|
+
should parse_line(sample2, 'another sample line').and_capture(
|
92
|
+
remote_host: '1.82.235.29', remote_logname: nil, user: nil,
|
93
|
+
timestamp: 20_090_908_075_405, http_status: 200, http_method: 'GET',
|
94
|
+
http_version: '1.1', bytes_sent: 23_414)
|
95
|
+
end
|
93
96
|
|
94
|
-
it { should_not parse_line('nonsense', 'a nonsense line')}
|
97
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
95
98
|
end
|
96
99
|
|
97
100
|
describe '#parse_io' do
|
98
101
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
99
102
|
|
100
|
-
it
|
103
|
+
it 'should parse a log snippet successfully without warnings' do
|
101
104
|
log_parser.should_receive(:handle_request).exactly(10).times
|
102
105
|
log_parser.should_not_receive(:warn)
|
103
106
|
log_parser.parse_file(log_fixture(:apache_common))
|
@@ -110,33 +113,35 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
110
113
|
|
111
114
|
it { should be_well_formed }
|
112
115
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent, :referer, :user_agent) }
|
113
|
-
it { should
|
116
|
+
it { should satisfy { |ff| ff.report_trackers.length == 8 } }
|
114
117
|
|
115
118
|
describe '#parse_line' do
|
116
119
|
let(:sample1) { '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"' }
|
117
120
|
let(:sample2) { '0:0:0:0:0:0:0:1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"' }
|
118
121
|
|
119
|
-
it
|
120
|
-
|
121
|
-
:
|
122
|
-
:
|
123
|
-
:
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
:
|
130
|
-
:
|
131
|
-
|
132
|
-
|
133
|
-
|
122
|
+
it do
|
123
|
+
should parse_line(sample1, 'with IPv4 address').and_capture(
|
124
|
+
remote_host: '69.41.0.45', remote_logname: nil, user: nil,
|
125
|
+
timestamp: 20_090_902_120_240, http_status: 404, http_method: 'GET',
|
126
|
+
http_version: '1.1', bytes_sent: 209, referer: nil,
|
127
|
+
user_agent: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
|
128
|
+
end
|
129
|
+
|
130
|
+
it do
|
131
|
+
should parse_line(sample2, 'with IPv6 address').and_capture(
|
132
|
+
remote_host: '0:0:0:0:0:0:0:1', remote_logname: nil, user: nil,
|
133
|
+
timestamp: 20_090_902_050_833, http_status: 200, http_method: 'GET',
|
134
|
+
http_version: '1.1', bytes_sent: 30, referer: nil,
|
135
|
+
user_agent: 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
|
136
|
+
end
|
137
|
+
|
138
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
134
139
|
end
|
135
140
|
|
136
141
|
describe '#parse_io' do
|
137
142
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
138
143
|
|
139
|
-
it
|
144
|
+
it 'should parse a log snippet successfully without warnings' do
|
140
145
|
log_parser.should_receive(:handle_request).exactly(5).times
|
141
146
|
log_parser.should_not_receive(:warn)
|
142
147
|
log_parser.parse_file(log_fixture(:apache_combined))
|
@@ -1,78 +1,75 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::CommonRegularExpressions do
|
4
|
-
|
4
|
+
|
5
5
|
include RequestLogAnalyzer::FileFormat::CommonRegularExpressions
|
6
6
|
|
7
7
|
describe '.timestamp' do
|
8
|
-
it
|
8
|
+
it 'should parse timestamps with a given format' do
|
9
9
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should =~ '2009-12-03T00:12:37+0100'
|
10
10
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03 00:12:37+0100'
|
11
11
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03T00:12:37'
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
-
|
16
15
|
describe '.hostname' do
|
17
|
-
it
|
16
|
+
it 'should parse hostnames successfully' do
|
18
17
|
anchored(hostname).should =~ 'railsdoctors.com'
|
19
18
|
anchored(hostname).should =~ 'www.rails-doctors.com'
|
20
19
|
anchored(hostname).should =~ 'hostname.co.uk'
|
21
20
|
anchored(hostname).should =~ 'localhost'
|
22
|
-
|
21
|
+
|
23
22
|
anchored(hostname).should_not =~ '192.168.0.1'
|
24
23
|
anchored(hostname).should_not =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
25
24
|
anchored(hostname).should_not =~ 'railsdoctors.'
|
26
25
|
end
|
27
26
|
end
|
28
|
-
|
27
|
+
|
29
28
|
describe '.ip_address' do
|
30
|
-
it
|
29
|
+
it 'should parse IPv4 addresses' do
|
31
30
|
anchored(ip_address).should =~ '127.0.0.1'
|
32
31
|
anchored(ip_address).should =~ '255.255.255.255'
|
33
|
-
|
32
|
+
|
34
33
|
anchored(ip_address).should_not =~ '2552.2552.2552.2552'
|
35
34
|
anchored(ip_address).should_not =~ '127001'
|
36
35
|
anchored(ip_address).should_not =~ ''
|
37
36
|
anchored(ip_address).should_not =~ '-'
|
38
37
|
anchored(ip_address).should_not =~ 'sub-host.domain.tld'
|
39
38
|
end
|
40
|
-
|
41
|
-
it
|
39
|
+
|
40
|
+
it 'should pase IPv6 addresses' do
|
42
41
|
anchored(ip_address).should =~ '::1'
|
43
42
|
anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
44
43
|
anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:127.0.0.1'
|
45
44
|
anchored(ip_address).should =~ '::3:200:f8ff:127.0.0.1'
|
46
45
|
anchored(ip_address).should =~ '0:0:0:0:0:0:0:1'
|
47
|
-
|
46
|
+
|
48
47
|
anchored(ip_address).should_not =~ 'qqqq:wwww:eeee:3q:200:wf8ff:fe21:67cf'
|
49
48
|
anchored(ip_address).should_not =~ '3ffe44:1900f:454545:3:200:f8ff:ffff:5432'
|
50
49
|
end
|
51
|
-
|
52
|
-
it
|
50
|
+
|
51
|
+
it 'should allow blank if true is given as parameter' do
|
53
52
|
anchored(ip_address(true)).should =~ ''
|
54
53
|
anchored(ip_address(true)).should_not =~ ' '
|
55
54
|
end
|
56
|
-
|
57
|
-
it
|
55
|
+
|
56
|
+
it 'should allow a nil substitute if a string is given as parameter' do
|
58
57
|
anchored(ip_address('-')).should =~ '-'
|
59
58
|
anchored(ip_address('-')).should_not =~ ' -'
|
60
59
|
anchored(ip_address('-')).should_not =~ '--'
|
61
60
|
anchored(ip_address('-')).should_not =~ ''
|
62
61
|
end
|
63
62
|
end
|
64
|
-
|
65
|
-
|
63
|
+
|
66
64
|
describe '.hostname_or_ip_address' do
|
67
|
-
it
|
65
|
+
it 'should parse either hostnames or ip addresses' do
|
68
66
|
anchored(hostname_or_ip_address).should =~ 'railsdoctors.com'
|
69
67
|
anchored(hostname_or_ip_address).should =~ 'hostname.co.uk'
|
70
68
|
anchored(hostname_or_ip_address).should =~ 'localhost'
|
71
69
|
anchored(hostname_or_ip_address).should =~ '192.168.0.1'
|
72
70
|
anchored(hostname_or_ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
73
|
-
|
71
|
+
|
74
72
|
anchored(hostname_or_ip_address).should_not =~ 'railsdoctors.'
|
75
73
|
end
|
76
|
-
end
|
74
|
+
end
|
77
75
|
end
|
78
|
-
|