request-log-analyzer 1.3.5 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +4 -3
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +13 -14
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/aggregator.rb +11 -15
- data/lib/request_log_analyzer/controller.rb +162 -89
- data/lib/request_log_analyzer/database/base.rb +21 -21
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +28 -103
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +17 -18
- data/lib/request_log_analyzer/file_format/apache.rb +26 -27
- data/lib/request_log_analyzer/file_format/merb.rb +30 -13
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +143 -73
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/file_format.rb +16 -28
- data/lib/request_log_analyzer/filter/anonymize.rb +8 -7
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/filter.rb +12 -16
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +27 -29
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/request.rb +75 -68
- data/lib/request_log_analyzer/source/database_loader.rb +10 -14
- data/lib/request_log_analyzer/source/log_parser.rb +57 -50
- data/lib/request_log_analyzer/source.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +21 -21
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/lib/request_log_analyzer/tracker.rb +139 -32
- data/lib/request_log_analyzer.rb +4 -4
- data/request-log-analyzer.gemspec +19 -15
- data/spec/database.yml +6 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +7 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +16 -16
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +26 -33
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +33 -38
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +117 -42
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +13 -8
|
@@ -1,49 +1,124 @@
|
|
|
1
1
|
require File.dirname(__FILE__) + '/../../spec_helper'
|
|
2
2
|
|
|
3
|
-
describe RequestLogAnalyzer::Tracker::Base
|
|
3
|
+
describe RequestLogAnalyzer::Tracker::Base do
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
5
|
+
describe 'API' do
|
|
6
|
+
|
|
7
|
+
before(:each) do
|
|
8
|
+
@tracker = Class.new(RequestLogAnalyzer::Tracker::Base).new
|
|
9
|
+
|
|
10
|
+
@summarizer = RequestLogAnalyzer::Aggregator::Summarizer.new(mock_source)
|
|
11
|
+
@summarizer.trackers << @tracker
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it "should receive :prepare when the summarizer is preparing" do
|
|
15
|
+
@tracker.should_receive(:prepare).once
|
|
16
|
+
@summarizer.prepare
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
it "should receive :update for every request for which should_update? returns true" do
|
|
20
|
+
@tracker.should_receive(:should_update?).twice.and_return(true)
|
|
21
|
+
@tracker.should_receive(:update).twice
|
|
22
|
+
|
|
23
|
+
@summarizer.aggregate(testing_format.request(:field => 'value1'))
|
|
24
|
+
@summarizer.aggregate(testing_format.request(:field => 'value2'))
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it "should not :update for every request for which should_update? returns false" do
|
|
28
|
+
@tracker.should_receive(:should_update?).twice.and_return(false)
|
|
29
|
+
@tracker.should_not_receive(:update)
|
|
30
|
+
|
|
31
|
+
@summarizer.aggregate(testing_format.request(:field => 'value1'))
|
|
32
|
+
@summarizer.aggregate(testing_format.request(:field => 'value2'))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it "should receive :report when the summary report is being built" do
|
|
36
|
+
@tracker.should_receive(:report).with(anything).once
|
|
37
|
+
@summarizer.report(mock_output)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it "should receieve :finalize when the summarizer is finalizing" do
|
|
41
|
+
@tracker.should_receive(:finalize).once
|
|
42
|
+
@summarizer.finalize
|
|
43
|
+
end
|
|
36
44
|
end
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
45
|
+
|
|
46
|
+
describe '#should_update?' do
|
|
47
|
+
before(:each) do
|
|
48
|
+
@tracker_class = Class.new(RequestLogAnalyzer::Tracker::Base)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
it "should return true by default, when no checks are installed" do
|
|
52
|
+
tracker = @tracker_class.new
|
|
53
|
+
tracker.should_update?(testing_format.request).should be_true
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
it "should return false if the line type is not in the request" do
|
|
57
|
+
tracker = @tracker_class.new(:line_type => :not_there)
|
|
58
|
+
tracker.should_update?(request(:line_type => :different)).should be_false
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
it "should return true if the line type is in the request" do
|
|
62
|
+
tracker = @tracker_class.new(:line_type => :there)
|
|
63
|
+
tracker.should_update?(request(:line_type => :there)).should be_true
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
it "should return true if a field name is given to :if and it is in the request" do
|
|
67
|
+
tracker = @tracker_class.new(:if => :field)
|
|
68
|
+
tracker.should_update?(request(:field => 'anything')).should be_true
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
it "should return false if a field name is given to :if and it is not the request" do
|
|
72
|
+
tracker = @tracker_class.new(:if => :field)
|
|
73
|
+
tracker.should_update?(request(:other_field => 'anything')).should be_false
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
it "should return false if a field name is given to :unless and it is in the request" do
|
|
77
|
+
tracker = @tracker_class.new(:unless => :field)
|
|
78
|
+
tracker.should_update?(request(:field => 'anything')).should be_false
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
it "should return true if a field name is given to :unless and it is not the request" do
|
|
82
|
+
tracker = @tracker_class.new(:unless => :field)
|
|
83
|
+
tracker.should_update?(request(:other_field => 'anything')).should be_true
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
it "should return the value of the block if one is given to the :if option" do
|
|
87
|
+
tracker = @tracker_class.new(:if => lambda { false } )
|
|
88
|
+
tracker.should_update?(request(:field => 'anything')).should be_false
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
it "should return the inverse value of the block if one is given to the :if option" do
|
|
92
|
+
tracker = @tracker_class.new(:unless => lambda { false } )
|
|
93
|
+
tracker.should_update?(request(:field => 'anything')).should be_true
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
it "should return false if any of the checks fail" do
|
|
97
|
+
tracker = @tracker_class.new(:if => :field, :unless => lambda { false }, :line_type => :not_present )
|
|
98
|
+
tracker.should_update?(request(:line_type => :present, :field => 'anything')).should be_false
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
it "should return true if all of the checks succeed" do
|
|
102
|
+
tracker = @tracker_class.new(:if => :field, :unless => lambda { false }, :line_type => :present )
|
|
103
|
+
tracker.should_update?(request(:line_type => :present, :field => 'anything')).should be_true
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
|
|
41
107
|
end
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
108
|
+
|
|
109
|
+
describe '#to_yaml_object' do
|
|
110
|
+
|
|
111
|
+
before(:each) do
|
|
112
|
+
@tracker = Class.new(RequestLogAnalyzer::Tracker::Base).new
|
|
113
|
+
|
|
114
|
+
@summarizer = RequestLogAnalyzer::Aggregator::Summarizer.new(mock_source)
|
|
115
|
+
@summarizer.trackers << @tracker
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
it "should receive :to_yaml object when finalizing" do
|
|
119
|
+
@summarizer.options[:dump] = temp_output_file(:dump)
|
|
120
|
+
@tracker.should_receive(:to_yaml_object).once
|
|
121
|
+
@summarizer.to_yaml
|
|
122
|
+
end
|
|
47
123
|
end
|
|
48
|
-
|
|
49
124
|
end
|
|
@@ -1,105 +1,107 @@
|
|
|
1
1
|
require File.dirname(__FILE__) + '/../../spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::Tracker::Traffic do
|
|
4
|
-
|
|
4
|
+
|
|
5
|
+
context 'using a field-based category' do
|
|
6
|
+
before(:each) do
|
|
7
|
+
@tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => :category)
|
|
8
|
+
@tracker.prepare
|
|
9
|
+
|
|
10
|
+
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
11
|
+
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
12
|
+
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
it "should register a request in the right category using the category field" do
|
|
16
|
+
@tracker.categories.should include('a', 'b')
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
it "should register requests under the correct category" do
|
|
20
|
+
@tracker.hits('a').should == 1
|
|
21
|
+
@tracker.hits('b').should == 2
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
context 'using a dynamic category' do
|
|
26
|
+
|
|
27
|
+
before(:each) do
|
|
28
|
+
@categorizer = lambda { |request| request[:traffic] < 2 ? 'few' : 'lots' }
|
|
29
|
+
@tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => @categorizer)
|
|
30
|
+
@tracker.prepare
|
|
31
|
+
|
|
32
|
+
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
33
|
+
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
34
|
+
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
it "should use the categorizer to determine the category" do
|
|
38
|
+
@tracker.categories.should include('few', 'lots')
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
it "should register requests under the correct category using the categorizer" do
|
|
42
|
+
@tracker.hits('few').should == 1
|
|
43
|
+
@tracker.hits('lots').should == 2
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
5
47
|
describe '#update' do
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
40
|
-
|
|
41
|
-
@tracker.average_traffic('a').should == 1.0
|
|
42
|
-
@tracker.average_traffic('b').should == 2.5
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
it "should set min and max traffic correctly" do
|
|
46
|
-
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
47
|
-
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
48
|
-
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
49
|
-
|
|
50
|
-
@tracker.min_traffic('b').should == 2
|
|
51
|
-
@tracker.max_traffic('b').should == 3
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
context 'using a dynamic category' do
|
|
57
|
-
before(:each) do
|
|
58
|
-
@categorizer = Proc.new { |request| request[:traffic] < 2 ? 'few' : 'lots' }
|
|
59
|
-
@tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => @categorizer)
|
|
60
|
-
@tracker.prepare
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
it "should use the categorizer to determine the right category" do
|
|
64
|
-
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
65
|
-
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
66
|
-
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
67
|
-
|
|
68
|
-
@tracker.categories.should include('few', 'lots')
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
it "should use the categorizer to aggregate the values correctly" do
|
|
72
|
-
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
73
|
-
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
74
|
-
@tracker.update(request(:category => 'b', :traffic => 3))
|
|
75
|
-
|
|
76
|
-
@tracker.max_traffic('few').should == 1
|
|
77
|
-
@tracker.min_traffic('lots').should == 2
|
|
78
|
-
end
|
|
48
|
+
|
|
49
|
+
before(:each) do
|
|
50
|
+
@tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => :category)
|
|
51
|
+
@tracker.prepare
|
|
52
|
+
|
|
53
|
+
@tracker.update(request(:category => 'a', :traffic => 2))
|
|
54
|
+
@tracker.update(request(:category => 'a', :traffic => 1))
|
|
55
|
+
@tracker.update(request(:category => 'a', :traffic => 3))
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it "should calculate the total traffic correctly" do
|
|
59
|
+
@tracker.sum('a').should == 6
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it "should calculate the traffic variance correctly" do
|
|
63
|
+
@tracker.variance('a').should == 1.0
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
it "should calculate the traffic standard deviation correctly" do
|
|
67
|
+
@tracker.stddev('a').should == 1.0
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it "should calculate the average traffic correctly" do
|
|
71
|
+
@tracker.mean('a').should == 2.0
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
it "should calculate the overall mean traffic correctly" do
|
|
75
|
+
@tracker.mean_overall.should == 2.0
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it "should set min and max traffic correctly" do
|
|
79
|
+
@tracker.min('a').should == 1
|
|
80
|
+
@tracker.max('a').should == 3
|
|
79
81
|
end
|
|
80
82
|
end
|
|
81
|
-
|
|
83
|
+
|
|
82
84
|
describe '#report' do
|
|
83
85
|
before(:each) do
|
|
84
86
|
@tracker = RequestLogAnalyzer::Tracker::Traffic.new(:category => :category, :traffic => :traffic)
|
|
85
87
|
@tracker.prepare
|
|
86
88
|
end
|
|
87
|
-
|
|
89
|
+
|
|
88
90
|
it "should generate a report without errors when one category is present" do
|
|
89
91
|
@tracker.update(request(:category => 'a', :traffic => 2))
|
|
90
92
|
@tracker.report(mock_output)
|
|
91
93
|
lambda { @tracker.report(mock_output) }.should_not raise_error
|
|
92
94
|
end
|
|
93
|
-
|
|
95
|
+
|
|
94
96
|
it "should generate a report without errors when no category is present" do
|
|
95
97
|
lambda { @tracker.report(mock_output) }.should_not raise_error
|
|
96
98
|
end
|
|
97
|
-
|
|
99
|
+
|
|
98
100
|
it "should generate a report without errors when multiple categories are present" do
|
|
99
101
|
@tracker.update(request(:category => 'a', :traffic => 2))
|
|
100
102
|
@tracker.update(request(:category => 'b', :traffic => 2))
|
|
101
103
|
lambda { @tracker.report(mock_output) }.should_not raise_error
|
|
102
104
|
end
|
|
103
|
-
|
|
105
|
+
|
|
104
106
|
end
|
|
105
107
|
end
|