request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -16,7 +16,7 @@ describe RequestLogAnalyzer::Database do
|
|
|
16
16
|
# FileFormat-agnostic classes
|
|
17
17
|
default_orm_class_names.each do |const|
|
|
18
18
|
it "should create the default #{const} constant" do
|
|
19
|
-
RequestLogAnalyzer::Database.const_defined?(const).should
|
|
19
|
+
RequestLogAnalyzer::Database.const_defined?(const).should == true
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
it "should create the default #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
|
|
@@ -25,9 +25,9 @@ describe RequestLogAnalyzer::Database do
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
# Some Fileformat-specific classes
|
|
28
|
-
|
|
28
|
+
%w(CompletedLine ProcessingLine).each do |const|
|
|
29
29
|
it "should create the #{const} constant" do
|
|
30
|
-
Object.const_defined?(const).should
|
|
30
|
+
Object.const_defined?(const).should == true
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
it "should create the #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
|
|
@@ -73,11 +73,11 @@ describe RequestLogAnalyzer::Database do
|
|
|
73
73
|
end
|
|
74
74
|
end
|
|
75
75
|
|
|
76
|
-
testing_format.line_definitions.each do |name,
|
|
76
|
+
testing_format.line_definitions.each do |name, _definition|
|
|
77
77
|
|
|
78
78
|
it "should create the #{(name.to_s + '_line').camelize} class for #{name.inspect} lines" do
|
|
79
79
|
@database.send :create_database_schema!
|
|
80
|
-
Object.const_defined?("#{name}_line".camelize).should
|
|
80
|
+
Object.const_defined?("#{name}_line".camelize).should == true
|
|
81
81
|
end
|
|
82
82
|
|
|
83
83
|
it "should create the #{name.to_s + '_lines'} table for the parsed #{name.inspect} lines" do
|
|
@@ -106,27 +106,27 @@ describe RequestLogAnalyzer::Database do
|
|
|
106
106
|
|
|
107
107
|
after(:each) { @database.remove_orm_classes! }
|
|
108
108
|
|
|
109
|
-
it
|
|
109
|
+
it 'should call :subclass_from_table when a table name is given as string' do
|
|
110
110
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
|
|
111
111
|
@database.load_activerecord_class('test_lines')
|
|
112
112
|
end
|
|
113
113
|
|
|
114
|
-
it
|
|
114
|
+
it 'should call :subclass_from_table when a table name is given as symbol' do
|
|
115
115
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
|
|
116
116
|
@database.load_activerecord_class(:test_lines)
|
|
117
117
|
end
|
|
118
118
|
|
|
119
|
-
it
|
|
119
|
+
it 'should call :subclass_from_table when a LineDefinition is given' do
|
|
120
120
|
RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_line_definition).and_return(@mock_class)
|
|
121
121
|
@database.load_activerecord_class(RequestLogAnalyzer::LineDefinition.new(:test))
|
|
122
122
|
end
|
|
123
123
|
|
|
124
|
-
it
|
|
124
|
+
it 'should define the class in the ORM module' do
|
|
125
125
|
@database.load_activerecord_class(:test_lines)
|
|
126
|
-
Object.const_defined?('TestLine').should
|
|
126
|
+
Object.const_defined?('TestLine').should == true
|
|
127
127
|
end
|
|
128
128
|
|
|
129
|
-
it
|
|
129
|
+
it 'should add the class to the line_classes array of the database' do
|
|
130
130
|
@database.load_activerecord_class(:test_lines)
|
|
131
131
|
@database.line_classes.should include(TestLine)
|
|
132
132
|
end
|
|
@@ -5,76 +5,80 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
|
|
6
6
|
|
|
7
7
|
it { should be_well_formed }
|
|
8
|
-
it
|
|
9
|
-
|
|
8
|
+
it do
|
|
9
|
+
should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
|
|
10
|
+
:key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent)
|
|
11
|
+
end
|
|
10
12
|
|
|
11
|
-
it { should
|
|
13
|
+
it { should satisfy { |ff| ff.report_trackers.length == 7 } }
|
|
12
14
|
|
|
13
15
|
let(:sample_get) { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
|
|
14
16
|
let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
|
|
15
17
|
let(:sample_2013) { 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac testbucket [17/Sep/2013:15:53:53 +0000] 87.193.165.87 - A9774F82F053FACE REST.GET.OBJECT somefile.txt "GET /testbucket/somefile.txt HTTP/1.1" 304 - - 1071 9 - "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36" -' }
|
|
16
18
|
|
|
17
19
|
describe '#parse_line' do
|
|
18
|
-
it
|
|
20
|
+
it do
|
|
19
21
|
should parse_line(sample_get, 'a GET line').and_capture(
|
|
20
|
-
:
|
|
21
|
-
:
|
|
22
|
-
:
|
|
23
|
-
:
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
:
|
|
27
|
-
:
|
|
28
|
-
:
|
|
29
|
-
:
|
|
30
|
-
:
|
|
31
|
-
:
|
|
32
|
-
:
|
|
33
|
-
:
|
|
34
|
-
:
|
|
35
|
-
:
|
|
36
|
-
:
|
|
37
|
-
|
|
22
|
+
bucket_owner: '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
|
|
23
|
+
bucket: 'sample-bucket',
|
|
24
|
+
timestamp: 20_061_006_014_214,
|
|
25
|
+
remote_ip: '207.171.172.6',
|
|
26
|
+
key: 'object.png',
|
|
27
|
+
operation: 'REST.GET.OBJECT',
|
|
28
|
+
requester: '65a011a29cdf8ec533ec3d1ccaae921c',
|
|
29
|
+
request_id: 'C980091AD89C936A',
|
|
30
|
+
request_uri: 'GET /sample-bucket/object.png HTTP/1.1',
|
|
31
|
+
error_code: nil,
|
|
32
|
+
http_status: 200,
|
|
33
|
+
total_time: 0.988,
|
|
34
|
+
turnaround_time: 0.987,
|
|
35
|
+
bytes_sent: 1243,
|
|
36
|
+
object_size: 1243,
|
|
37
|
+
user_agent: 'aranhabot',
|
|
38
|
+
referer: nil)
|
|
39
|
+
end
|
|
38
40
|
|
|
39
|
-
it
|
|
40
|
-
|
|
41
|
-
:
|
|
42
|
-
:
|
|
43
|
-
:
|
|
44
|
-
:
|
|
45
|
-
:
|
|
46
|
-
:
|
|
47
|
-
:
|
|
48
|
-
:
|
|
49
|
-
:
|
|
50
|
-
:
|
|
51
|
-
:
|
|
52
|
-
:
|
|
53
|
-
:
|
|
54
|
-
:
|
|
55
|
-
:
|
|
56
|
-
:
|
|
57
|
-
|
|
41
|
+
it do
|
|
42
|
+
should parse_line(sample_2013, '2013 sample').and_capture(
|
|
43
|
+
bucket_owner: 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac',
|
|
44
|
+
bucket: 'testbucket',
|
|
45
|
+
timestamp: 20_130_917_155_353,
|
|
46
|
+
remote_ip: '87.193.165.87',
|
|
47
|
+
requester: '-',
|
|
48
|
+
request_id: 'A9774F82F053FACE',
|
|
49
|
+
operation: 'REST.GET.OBJECT',
|
|
50
|
+
key: 'somefile.txt',
|
|
51
|
+
request_uri: 'GET /testbucket/somefile.txt HTTP/1.1',
|
|
52
|
+
http_status: 304,
|
|
53
|
+
error_code: nil,
|
|
54
|
+
bytes_sent: 0,
|
|
55
|
+
object_size: 1071,
|
|
56
|
+
total_time: 0.009,
|
|
57
|
+
turnaround_time: 0,
|
|
58
|
+
referer: 'https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US',
|
|
59
|
+
user_agent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36')
|
|
60
|
+
end
|
|
58
61
|
|
|
59
|
-
it
|
|
60
|
-
|
|
61
|
-
:
|
|
62
|
-
:
|
|
63
|
-
:
|
|
64
|
-
:
|
|
65
|
-
:
|
|
66
|
-
:
|
|
67
|
-
:
|
|
68
|
-
:
|
|
69
|
-
:
|
|
70
|
-
:
|
|
71
|
-
:
|
|
72
|
-
:
|
|
73
|
-
:
|
|
74
|
-
:
|
|
75
|
-
:
|
|
76
|
-
:
|
|
77
|
-
|
|
62
|
+
it do
|
|
63
|
+
should parse_line(sample_copy, 'a COPY line').and_capture(
|
|
64
|
+
bucket_owner: '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
65
|
+
bucket: 'testbucket',
|
|
66
|
+
timestamp: 20_100_303_230_459,
|
|
67
|
+
remote_ip: '174.119.31.76',
|
|
68
|
+
key: 'files/image.png',
|
|
69
|
+
operation: 'REST.COPY.OBJECT',
|
|
70
|
+
requester: '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
71
|
+
request_id: 'ACCC34B843C87BC9',
|
|
72
|
+
request_uri: 'PUT /files/image.png HTTP/1.1',
|
|
73
|
+
error_code: nil,
|
|
74
|
+
http_status: 200,
|
|
75
|
+
total_time: 0.365,
|
|
76
|
+
turnaround_time: 0.319,
|
|
77
|
+
bytes_sent: 234,
|
|
78
|
+
object_size: 65_957,
|
|
79
|
+
user_agent: '',
|
|
80
|
+
referer: nil)
|
|
81
|
+
end
|
|
78
82
|
|
|
79
83
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
80
84
|
end
|
|
@@ -83,10 +87,10 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
|
83
87
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
84
88
|
let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
|
|
85
89
|
|
|
86
|
-
it
|
|
90
|
+
it 'should parse requests correctly and not generate warnings' do
|
|
87
91
|
log_parser.should_receive(:handle_request).twice
|
|
88
92
|
log_parser.should_not_receive(:warn)
|
|
89
93
|
log_parser.parse_io(snippet)
|
|
90
94
|
end
|
|
91
95
|
end
|
|
92
|
-
end
|
|
96
|
+
end
|
|
@@ -6,41 +6,41 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
|
6
6
|
let(:format_string) { '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T' }
|
|
7
7
|
let(:line_definition) { RequestLogAnalyzer::FileFormat::Apache.access_line_definition(format_string) }
|
|
8
8
|
|
|
9
|
-
it
|
|
9
|
+
it 'should create a Regexp to match the line' do
|
|
10
10
|
line_definition.regexp.should be_kind_of(Regexp)
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
it
|
|
14
|
-
line_definition.captures.should
|
|
13
|
+
it 'should create a list of captures for the values in the lines' do
|
|
14
|
+
line_definition.captures.length.should == 12
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
it
|
|
17
|
+
it 'should make it a header line' do
|
|
18
18
|
line_definition.should be_header
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
it
|
|
21
|
+
it 'should make it a footer line' do
|
|
22
22
|
line_definition.should be_footer
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
it
|
|
26
|
-
line_definition.captures?(:duration).should
|
|
25
|
+
it 'should capture :duration' do
|
|
26
|
+
line_definition.captures?(:duration).should == true
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
describe '.access_line_definition' do
|
|
31
|
-
it
|
|
31
|
+
it 'should parse values in microseconds when no argument is given to %D' do
|
|
32
32
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
|
|
33
|
-
format.should parse_line('12345').and_capture(:
|
|
33
|
+
format.should parse_line('12345').and_capture(duration: 0.012345)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
it
|
|
36
|
+
it 'should parse values in microseconds when micro is given as argument to %D' do
|
|
37
37
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
|
|
38
|
-
format.should parse_line('12345').and_capture(:
|
|
38
|
+
format.should parse_line('12345').and_capture(duration: 0.012345)
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
it
|
|
41
|
+
it 'should parse values in microseconds when micro is given as argument to %D' do
|
|
42
42
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
|
|
43
|
-
format.should parse_line('12345').and_capture(:
|
|
43
|
+
format.should parse_line('12345').and_capture(duration: 12.345)
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
46
|
|
|
@@ -50,20 +50,21 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
|
50
50
|
|
|
51
51
|
it { should be_well_formed }
|
|
52
52
|
it { should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status) }
|
|
53
|
-
it { should
|
|
53
|
+
it { should satisfy { |ff| ff.report_trackers.length == 8 } }
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
context '"vhost_combined" access log parsing' do
|
|
57
57
|
subject { RequestLogAnalyzer::FileFormat.load(:apache, :vhost_combined) }
|
|
58
58
|
describe '#parse_line' do
|
|
59
|
-
let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683'}
|
|
60
|
-
|
|
61
|
-
it
|
|
62
|
-
|
|
63
|
-
:
|
|
64
|
-
:
|
|
65
|
-
:
|
|
66
|
-
|
|
59
|
+
let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683' }
|
|
60
|
+
|
|
61
|
+
it do
|
|
62
|
+
should parse_line(sample1, 'a sample line').and_capture(
|
|
63
|
+
remote_host: '72.204.80.86', remote_logname: nil, user: nil,
|
|
64
|
+
vhost: 'psi-equipment.od1.vtiger.com', duration: 1.0,
|
|
65
|
+
timestamp: 20_131_019_154_154, http_status: 304, http_method: 'GET',
|
|
66
|
+
http_version: '1.1', bytes_sent: 0)
|
|
67
|
+
end
|
|
67
68
|
end
|
|
68
69
|
end
|
|
69
70
|
|
|
@@ -72,32 +73,34 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
|
72
73
|
|
|
73
74
|
it { should be_well_formed }
|
|
74
75
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent) }
|
|
75
|
-
it { should
|
|
76
|
+
it { should satisfy { |ff| ff.report_trackers.length == 6 } }
|
|
76
77
|
|
|
77
78
|
describe '#parse_line' do
|
|
78
79
|
|
|
79
80
|
let(:sample1) { '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223' }
|
|
80
81
|
let(:sample2) { '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414' }
|
|
81
82
|
|
|
82
|
-
it
|
|
83
|
-
|
|
84
|
-
:
|
|
85
|
-
:
|
|
86
|
-
|
|
83
|
+
it do
|
|
84
|
+
should parse_line(sample1, 'a sample line').and_capture(
|
|
85
|
+
remote_host: '1.129.119.13', remote_logname: nil, user: nil,
|
|
86
|
+
timestamp: 20_090_908_075_409, http_status: 200, http_method: 'GET',
|
|
87
|
+
http_version: '1.0', bytes_sent: 8223)
|
|
88
|
+
end
|
|
87
89
|
|
|
88
|
-
it
|
|
89
|
-
|
|
90
|
-
:
|
|
91
|
-
:
|
|
92
|
-
|
|
90
|
+
it do
|
|
91
|
+
should parse_line(sample2, 'another sample line').and_capture(
|
|
92
|
+
remote_host: '1.82.235.29', remote_logname: nil, user: nil,
|
|
93
|
+
timestamp: 20_090_908_075_405, http_status: 200, http_method: 'GET',
|
|
94
|
+
http_version: '1.1', bytes_sent: 23_414)
|
|
95
|
+
end
|
|
93
96
|
|
|
94
|
-
it { should_not parse_line('nonsense', 'a nonsense line')}
|
|
97
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
95
98
|
end
|
|
96
99
|
|
|
97
100
|
describe '#parse_io' do
|
|
98
101
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
99
102
|
|
|
100
|
-
it
|
|
103
|
+
it 'should parse a log snippet successfully without warnings' do
|
|
101
104
|
log_parser.should_receive(:handle_request).exactly(10).times
|
|
102
105
|
log_parser.should_not_receive(:warn)
|
|
103
106
|
log_parser.parse_file(log_fixture(:apache_common))
|
|
@@ -110,33 +113,35 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
|
110
113
|
|
|
111
114
|
it { should be_well_formed }
|
|
112
115
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent, :referer, :user_agent) }
|
|
113
|
-
it { should
|
|
116
|
+
it { should satisfy { |ff| ff.report_trackers.length == 8 } }
|
|
114
117
|
|
|
115
118
|
describe '#parse_line' do
|
|
116
119
|
let(:sample1) { '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"' }
|
|
117
120
|
let(:sample2) { '0:0:0:0:0:0:0:1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"' }
|
|
118
121
|
|
|
119
|
-
it
|
|
120
|
-
|
|
121
|
-
:
|
|
122
|
-
:
|
|
123
|
-
:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
:
|
|
130
|
-
:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
122
|
+
it do
|
|
123
|
+
should parse_line(sample1, 'with IPv4 address').and_capture(
|
|
124
|
+
remote_host: '69.41.0.45', remote_logname: nil, user: nil,
|
|
125
|
+
timestamp: 20_090_902_120_240, http_status: 404, http_method: 'GET',
|
|
126
|
+
http_version: '1.1', bytes_sent: 209, referer: nil,
|
|
127
|
+
user_agent: 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
it do
|
|
131
|
+
should parse_line(sample2, 'with IPv6 address').and_capture(
|
|
132
|
+
remote_host: '0:0:0:0:0:0:0:1', remote_logname: nil, user: nil,
|
|
133
|
+
timestamp: 20_090_902_050_833, http_status: 200, http_method: 'GET',
|
|
134
|
+
http_version: '1.1', bytes_sent: 30, referer: nil,
|
|
135
|
+
user_agent: 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
134
139
|
end
|
|
135
140
|
|
|
136
141
|
describe '#parse_io' do
|
|
137
142
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
138
143
|
|
|
139
|
-
it
|
|
144
|
+
it 'should parse a log snippet successfully without warnings' do
|
|
140
145
|
log_parser.should_receive(:handle_request).exactly(5).times
|
|
141
146
|
log_parser.should_not_receive(:warn)
|
|
142
147
|
log_parser.parse_file(log_fixture(:apache_combined))
|
|
@@ -1,78 +1,75 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::CommonRegularExpressions do
|
|
4
|
-
|
|
4
|
+
|
|
5
5
|
include RequestLogAnalyzer::FileFormat::CommonRegularExpressions
|
|
6
6
|
|
|
7
7
|
describe '.timestamp' do
|
|
8
|
-
it
|
|
8
|
+
it 'should parse timestamps with a given format' do
|
|
9
9
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should =~ '2009-12-03T00:12:37+0100'
|
|
10
10
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03 00:12:37+0100'
|
|
11
11
|
anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03T00:12:37'
|
|
12
12
|
end
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
|
|
16
15
|
describe '.hostname' do
|
|
17
|
-
it
|
|
16
|
+
it 'should parse hostnames successfully' do
|
|
18
17
|
anchored(hostname).should =~ 'railsdoctors.com'
|
|
19
18
|
anchored(hostname).should =~ 'www.rails-doctors.com'
|
|
20
19
|
anchored(hostname).should =~ 'hostname.co.uk'
|
|
21
20
|
anchored(hostname).should =~ 'localhost'
|
|
22
|
-
|
|
21
|
+
|
|
23
22
|
anchored(hostname).should_not =~ '192.168.0.1'
|
|
24
23
|
anchored(hostname).should_not =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
|
25
24
|
anchored(hostname).should_not =~ 'railsdoctors.'
|
|
26
25
|
end
|
|
27
26
|
end
|
|
28
|
-
|
|
27
|
+
|
|
29
28
|
describe '.ip_address' do
|
|
30
|
-
it
|
|
29
|
+
it 'should parse IPv4 addresses' do
|
|
31
30
|
anchored(ip_address).should =~ '127.0.0.1'
|
|
32
31
|
anchored(ip_address).should =~ '255.255.255.255'
|
|
33
|
-
|
|
32
|
+
|
|
34
33
|
anchored(ip_address).should_not =~ '2552.2552.2552.2552'
|
|
35
34
|
anchored(ip_address).should_not =~ '127001'
|
|
36
35
|
anchored(ip_address).should_not =~ ''
|
|
37
36
|
anchored(ip_address).should_not =~ '-'
|
|
38
37
|
anchored(ip_address).should_not =~ 'sub-host.domain.tld'
|
|
39
38
|
end
|
|
40
|
-
|
|
41
|
-
it
|
|
39
|
+
|
|
40
|
+
it 'should pase IPv6 addresses' do
|
|
42
41
|
anchored(ip_address).should =~ '::1'
|
|
43
42
|
anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
|
44
43
|
anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:127.0.0.1'
|
|
45
44
|
anchored(ip_address).should =~ '::3:200:f8ff:127.0.0.1'
|
|
46
45
|
anchored(ip_address).should =~ '0:0:0:0:0:0:0:1'
|
|
47
|
-
|
|
46
|
+
|
|
48
47
|
anchored(ip_address).should_not =~ 'qqqq:wwww:eeee:3q:200:wf8ff:fe21:67cf'
|
|
49
48
|
anchored(ip_address).should_not =~ '3ffe44:1900f:454545:3:200:f8ff:ffff:5432'
|
|
50
49
|
end
|
|
51
|
-
|
|
52
|
-
it
|
|
50
|
+
|
|
51
|
+
it 'should allow blank if true is given as parameter' do
|
|
53
52
|
anchored(ip_address(true)).should =~ ''
|
|
54
53
|
anchored(ip_address(true)).should_not =~ ' '
|
|
55
54
|
end
|
|
56
|
-
|
|
57
|
-
it
|
|
55
|
+
|
|
56
|
+
it 'should allow a nil substitute if a string is given as parameter' do
|
|
58
57
|
anchored(ip_address('-')).should =~ '-'
|
|
59
58
|
anchored(ip_address('-')).should_not =~ ' -'
|
|
60
59
|
anchored(ip_address('-')).should_not =~ '--'
|
|
61
60
|
anchored(ip_address('-')).should_not =~ ''
|
|
62
61
|
end
|
|
63
62
|
end
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
|
|
66
64
|
describe '.hostname_or_ip_address' do
|
|
67
|
-
it
|
|
65
|
+
it 'should parse either hostnames or ip addresses' do
|
|
68
66
|
anchored(hostname_or_ip_address).should =~ 'railsdoctors.com'
|
|
69
67
|
anchored(hostname_or_ip_address).should =~ 'hostname.co.uk'
|
|
70
68
|
anchored(hostname_or_ip_address).should =~ 'localhost'
|
|
71
69
|
anchored(hostname_or_ip_address).should =~ '192.168.0.1'
|
|
72
70
|
anchored(hostname_or_ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
|
|
73
|
-
|
|
71
|
+
|
|
74
72
|
anchored(hostname_or_ip_address).should_not =~ 'railsdoctors.'
|
|
75
73
|
end
|
|
76
|
-
end
|
|
74
|
+
end
|
|
77
75
|
end
|
|
78
|
-
|