request-log-analyzer 1.3.7 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +2 -2
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer.rb +4 -4
  10. data/lib/request_log_analyzer/aggregator.rb +10 -10
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
  12. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  14. data/lib/request_log_analyzer/controller.rb +153 -69
  15. data/lib/request_log_analyzer/database.rb +13 -13
  16. data/lib/request_log_analyzer/database/base.rb +17 -17
  17. data/lib/request_log_analyzer/database/connection.rb +3 -3
  18. data/lib/request_log_analyzer/database/request.rb +2 -2
  19. data/lib/request_log_analyzer/database/source.rb +1 -1
  20. data/lib/request_log_analyzer/file_format.rb +15 -15
  21. data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
  22. data/lib/request_log_analyzer/file_format/apache.rb +20 -19
  23. data/lib/request_log_analyzer/file_format/merb.rb +12 -12
  24. data/lib/request_log_analyzer/file_format/rack.rb +4 -4
  25. data/lib/request_log_analyzer/file_format/rails.rb +146 -70
  26. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  27. data/lib/request_log_analyzer/filter.rb +6 -6
  28. data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
  29. data/lib/request_log_analyzer/filter/field.rb +9 -9
  30. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +22 -22
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output.rb +53 -12
  35. data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
  36. data/lib/request_log_analyzer/output/html.rb +20 -20
  37. data/lib/request_log_analyzer/request.rb +35 -36
  38. data/lib/request_log_analyzer/source.rb +7 -7
  39. data/lib/request_log_analyzer/source/database_loader.rb +7 -7
  40. data/lib/request_log_analyzer/source/log_parser.rb +48 -43
  41. data/lib/request_log_analyzer/tracker.rb +128 -14
  42. data/lib/request_log_analyzer/tracker/duration.rb +39 -132
  43. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  44. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
  45. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  46. data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
  47. data/request-log-analyzer.gemspec +19 -15
  48. data/spec/fixtures/rails_22.log +1 -1
  49. data/spec/integration/command_line_usage_spec.rb +1 -1
  50. data/spec/lib/helpers.rb +7 -7
  51. data/spec/lib/macros.rb +3 -3
  52. data/spec/lib/matchers.rb +41 -27
  53. data/spec/lib/mocks.rb +15 -14
  54. data/spec/lib/testing_format.rb +9 -9
  55. data/spec/spec_helper.rb +6 -6
  56. data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
  57. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  58. data/spec/unit/controller/controller_spec.rb +2 -2
  59. data/spec/unit/controller/log_processor_spec.rb +1 -1
  60. data/spec/unit/database/base_class_spec.rb +19 -19
  61. data/spec/unit/database/connection_spec.rb +3 -3
  62. data/spec/unit/database/database_spec.rb +25 -25
  63. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  64. data/spec/unit/file_format/apache_format_spec.rb +13 -13
  65. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  66. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  67. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  68. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  69. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  70. data/spec/unit/filter/field_filter_spec.rb +13 -13
  71. data/spec/unit/filter/filter_spec.rb +1 -1
  72. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  73. data/spec/unit/mailer_spec.rb +30 -0
  74. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  75. data/spec/unit/source/log_parser_spec.rb +27 -27
  76. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  77. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  78. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  79. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  80. data/spec/unit/tracker/tracker_api_spec.rb +13 -13
  81. data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
  82. data/tasks/github-gem.rake +125 -75
  83. data/tasks/request_log_analyzer.rake +2 -2
  84. metadata +8 -6
@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Anonymize.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should anonimize ip" do
10
10
  @filter.filter(request(:ip => '123.123.123.123'))[:ip].should_not eql('123.123.123.123')
11
11
  end
@@ -17,5 +17,5 @@ describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
17
17
  it "should fuzz durations" do
18
18
  @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
19
19
  end
20
-
20
+
21
21
  end
@@ -5,22 +5,22 @@ describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :select)
7
7
  end
8
-
8
+
9
9
  it "should reject a request if the field value does not match" do
10
10
  @filter.filter(request(:test => 'not test')).should be_nil
11
11
  end
12
-
12
+
13
13
  it "should reject a request if the field name does not match" do
14
14
  @filter.filter(request(:testing => 'test')).should be_nil
15
15
  end
16
16
 
17
17
  it "should accept a request if the both name and value match" do
18
18
  @filter.filter(request(:test => 'test')).should_not be_nil
19
- end
20
-
19
+ end
20
+
21
21
  it "should accept a request if the value is not the first value" do
22
22
  @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
23
- end
23
+ end
24
24
  end
25
25
 
26
26
  describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
@@ -28,22 +28,22 @@ describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
28
28
  before(:each) do
29
29
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :reject)
30
30
  end
31
-
31
+
32
32
  it "should accept a request if the field value does not match" do
33
33
  @filter.filter(request(:test => 'not test')).should_not be_nil
34
34
  end
35
-
35
+
36
36
  it "should accept a request if the field name does not match" do
37
37
  @filter.filter(request(:testing => 'test')).should_not be_nil
38
38
  end
39
39
 
40
40
  it "should reject a request if the both name and value match" do
41
41
  @filter.filter(request(:test => 'test')).should be_nil
42
- end
43
-
42
+ end
43
+
44
44
  it "should reject a request if the value is not the first value" do
45
45
  @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
46
- end
46
+ end
47
47
  end
48
48
 
49
49
  describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
@@ -51,16 +51,16 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
51
51
  before(:each) do
52
52
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => '/test/', :mode => :select)
53
53
  end
54
-
54
+
55
55
  it "should reject a request if the field value does not match" do
56
56
  @filter.filter(request(:test => 'a working test')).should_not be_nil
57
57
  end
58
-
58
+
59
59
  it "should reject a request if the field name does not match" do
60
60
  @filter.filter(request(:testing => 'test')).should be_nil
61
61
  end
62
62
 
63
63
  it "should accept a request if the value is not the first value" do
64
64
  @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
65
- end
65
+ end
66
66
  end
@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::Filter::Base, 'base filter' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Base.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should return everything" do
10
10
  @filter.filter(request(:ip => '123.123.123.123'))[:ip].should eql('123.123.123.123')
11
11
  end
@@ -5,54 +5,54 @@ describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
7
7
  end
8
-
8
+
9
9
  it "should reject a request before the after date" do
10
10
  @filter.filter(request(:timestamp => 20081212000000)).should be_nil
11
11
  end
12
-
12
+
13
13
  it "should reject a request after the before date" do
14
14
  @filter.filter(request(:timestamp => 20090303000000)).should be_nil
15
15
  end
16
-
16
+
17
17
  it "should accept a request between the after and before dates" do
18
18
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
19
19
  end
20
20
  end
21
21
 
22
22
  describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
23
-
23
+
24
24
  before(:each) do
25
- @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :before => DateTime.parse('2009-02-02'))
25
+ @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :before => DateTime.parse('2009-02-02'))
26
26
  end
27
-
27
+
28
28
  it "should accept a request before the after date" do
29
29
  @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
30
30
  end
31
-
31
+
32
32
  it "should reject a request after the before date" do
33
33
  @filter.filter(request(:timestamp => 20090303000000)).should be_nil
34
34
  end
35
-
35
+
36
36
  it "should accept a request between the after and before dates" do
37
37
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
38
- end
38
+ end
39
39
  end
40
40
 
41
- describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
42
-
41
+ describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
42
+
43
43
  before(:each) do
44
44
  @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'))
45
45
  end
46
-
46
+
47
47
  it "should reject a request before the after date" do
48
48
  @filter.filter(request(:timestamp => 20081212000000)).should be_nil
49
49
  end
50
-
50
+
51
51
  it "should accept a request after the before date" do
52
52
  @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
53
53
  end
54
-
54
+
55
55
  it "should accept a request between the after and before dates" do
56
56
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
57
- end
57
+ end
58
58
  end
@@ -0,0 +1,30 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ describe RequestLogAnalyzer::Mailer, 'mailer' do
4
+
5
+ it "should store printed data" do
6
+ @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', :debug => true)
7
+
8
+ @mailer << 'test1'
9
+ @mailer.puts 'test2'
10
+
11
+ @mailer.data.should eql(['test1', 'test2'])
12
+ end
13
+
14
+ it "should send mail" do
15
+ @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', :debug => true)
16
+
17
+ @mailer << 'test1'
18
+ @mailer.puts 'test2'
19
+
20
+ mail = @mailer.mail
21
+
22
+ mail[0].should include("contact@railsdoctors.com")
23
+ mail[0].should include("test1")
24
+ mail[0].should include("test2")
25
+
26
+ mail[1].should include("contact@railsdoctors.com")
27
+ mail[2].should include("alfa@beta.com")
28
+ end
29
+
30
+ end
@@ -1,49 +1,49 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Request do
4
-
4
+
5
5
  before(:each) do
6
6
  @request = testing_format.request
7
- end
8
-
7
+ end
8
+
9
9
  it "should be empty without any captured lines in it" do
10
10
  @request.should be_empty
11
11
  end
12
-
12
+
13
13
  context :incomplete do
14
-
14
+
15
15
  before(:each) do
16
16
  @request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
17
17
  end
18
-
18
+
19
19
  it "should be single if only one line has been added" do
20
20
  @request.should_not be_empty
21
21
  end
22
-
22
+
23
23
  it "should not be a completed request" do
24
24
  @request.should_not be_completed
25
- end
26
-
25
+ end
26
+
27
27
  it "should take the line type of the first line as global line_type" do
28
28
  @request.lines[0][:line_type].should == :test
29
29
  @request.should =~ :test
30
30
  end
31
-
31
+
32
32
  it "should return the first field value" do
33
33
  @request[:test_capture].should == 'awesome!'
34
34
  end
35
-
35
+
36
36
  it "should return nil if no such field is present" do
37
37
  @request[:nonexisting].should be_nil
38
38
  end
39
39
  end
40
-
40
+
41
41
  context :completed do
42
-
42
+
43
43
  before(:each) do
44
- @request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
45
- @request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
46
- @request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
44
+ @request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
45
+ @request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
46
+ @request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
47
47
  @request << { :line_type => :last, :lineno => 10, :time => 0.03 }
48
48
  end
49
49
 
@@ -64,29 +64,29 @@ describe RequestLogAnalyzer::Request do
64
64
  @request[:time].should == 0.03
65
65
  end
66
66
 
67
- it "should detect the first matching field value" do
67
+ it "should detect the first matching field value" do
68
68
  @request.first(:test_capture).should == 'testing'
69
69
  end
70
70
 
71
- it "should detect the every matching field value" do
71
+ it "should detect the every matching field value" do
72
72
  @request.every(:test_capture).should == ['testing', "testing some more"]
73
73
  end
74
74
 
75
75
  it "should set the first_lineno for a request to the lowest lineno encountered" do
76
76
  @request.first_lineno.should eql(1)
77
77
  end
78
-
78
+
79
79
  it "should set the first_lineno for a request if a line with a lower lineno is added" do
80
- @request << { :line_type => :test, :lineno => 0 }
80
+ @request << { :line_type => :test, :lineno => 0 }
81
81
  @request.first_lineno.should eql(0)
82
- end
82
+ end
83
83
 
84
84
  it "should set the last_lineno for a request to the highest encountered lineno" do
85
85
  @request.last_lineno.should eql(10)
86
86
  end
87
-
87
+
88
88
  it "should not set the last_lineno for a request if a line with a lower lineno is added" do
89
- @request << { :line_type => :test, :lineno => 7 }
89
+ @request << { :line_type => :test, :lineno => 7 }
90
90
  @request.last_lineno.should eql(10)
91
91
  end
92
92
 
@@ -95,17 +95,17 @@ describe RequestLogAnalyzer::Request do
95
95
  end
96
96
 
97
97
  it "should set return a timestamp field if such a field is captured" do
98
- @request << { :line_type => :first, :lineno => 1, :name => 'first line!', :timestamp => Time.now}
98
+ @request << { :line_type => :first, :lineno => 1, :name => 'first line!', :timestamp => Time.now}
99
99
  @request.timestamp.should_not be_nil
100
- end
100
+ end
101
101
  end
102
-
102
+
103
103
  context 'single line' do
104
104
  # combined is both a header and a footer line
105
105
  before(:each) { @request << { :line_type => :combined, :lineno => 1 } }
106
-
106
+
107
107
  it "should be a completed request if the line is both header and footer" do
108
108
  @request.should be_completed
109
- end
109
+ end
110
110
  end
111
- end
111
+ end
@@ -1,60 +1,60 @@
1
1
  require File.dirname(__FILE__) + '/../../spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Source::LogParser, :requests do
4
-
4
+
5
5
  before(:each) do
6
6
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should have multiple line definitions" do
10
10
  @log_parser.file_format.line_definitions.length.should >= 2
11
- end
12
-
11
+ end
12
+
13
13
  it "should have a valid language" do
14
14
  @log_parser.file_format.should be_valid
15
15
  end
16
-
16
+
17
17
  it "should set the :source for every parsed line" do
18
18
  @log_parser.parse_file(log_fixture(:rails_22)) do |request|
19
19
  request.lines.all? { |line| line[:source] == log_fixture(:rails_22) }.should be_true
20
20
  end
21
21
  end
22
-
22
+
23
23
  it "should set the :lineno for every parsed line" do
24
24
  @log_parser.parse_file(log_fixture(:rails_22)) do |request|
25
25
  request.lines.all? { |line| line.has_key?(:lineno) }.should be_true
26
26
  end
27
27
  end
28
-
28
+
29
29
  it "should parse more lines than requests" do
30
30
  @log_parser.should_receive(:handle_request).with(an_instance_of(TestingFormat::Request)).twice
31
31
  @log_parser.parse_file(log_fixture(:test_language_combined))
32
- @log_parser.parsed_lines.should > 2
32
+ @log_parser.parsed_lines.should > 2
33
33
  end
34
-
34
+
35
35
  it "should parse requests spanned over multiple files" do
36
36
  @log_parser.should_receive(:handle_request).with(an_instance_of(TestingFormat::Request)).once
37
37
  @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
38
38
  end
39
-
39
+
40
40
  it "should parse all request values when spanned over multiple files" do
41
41
  @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
42
- request.lines.should have(4).items
42
+ request.lines.should have(4).items
43
43
  request[:request_no].should == 1
44
- request[:test_capture].should == "Testing is amazing" # Note the custom converter
44
+ request[:test_capture].should == "Testing is amazing" # Note the custom converter
45
45
  end
46
- end
47
-
46
+ end
47
+
48
48
  it "should parse a stream and find valid requests" do
49
49
  io = File.new(log_fixture(:test_file_format), 'r')
50
50
  @log_parser.parse_io(io) do |request|
51
51
  request.should be_kind_of(RequestLogAnalyzer::Request)
52
52
  request.should =~ :test
53
- request[:test_capture].should_not be_nil
53
+ request[:test_capture].should_not be_nil
54
54
  end
55
55
  io.close
56
56
  end
57
-
57
+
58
58
  it "should parse a request that only consists of one line" do
59
59
  @log_parser.parse_file(log_fixture(:header_and_footer))
60
60
  @log_parser.parsed_requests.should == 2
@@ -62,30 +62,30 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
62
62
  end
63
63
 
64
64
  describe RequestLogAnalyzer::Source::LogParser, :warnings do
65
-
65
+
66
66
  before(:each) do
67
67
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format, :parse_strategy => 'cautious')
68
68
  end
69
-
69
+
70
70
  it "should warn about teaser matching problems" do
71
71
  @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
72
72
  @log_parser.parse_file(log_fixture(:test_file_format))
73
73
  end
74
-
74
+
75
75
  it "should warn about unmatching request headers and footers" do
76
- @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
77
- @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
78
- @log_parser.should_not_receive(:handle_request)
79
- @log_parser.parse_file(log_fixture(:test_order))
80
- end
76
+ @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
77
+ @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
78
+ @log_parser.should_not_receive(:handle_request)
79
+ @log_parser.parse_file(log_fixture(:test_order))
80
+ end
81
81
  end
82
82
 
83
83
  describe RequestLogAnalyzer::Source::LogParser, :decompression do
84
-
84
+
85
85
  before(:each) do
86
86
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat::Rails.create)
87
87
  end
88
-
88
+
89
89
  it "should parse a rails gzipped log file" do
90
90
  @log_parser.should_receive(:handle_request).once
91
91
  @log_parser.parse_file(log_fixture(:decompression, "log.gz"))
@@ -115,5 +115,5 @@ describe RequestLogAnalyzer::Source::LogParser, :decompression do
115
115
  @log_parser.parse_file(log_fixture(:decompression, "log.zip"))
116
116
  @log_parser.parsed_lines.should > 0
117
117
  end
118
-
118
+
119
119
  end