request-log-analyzer 1.3.7 → 1.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. data/LICENSE +3 -3
  2. data/README.rdoc +1 -1
  3. data/bin/request-log-analyzer +17 -14
  4. data/lib/cli/command_line_arguments.rb +51 -51
  5. data/lib/cli/database_console.rb +3 -3
  6. data/lib/cli/database_console_init.rb +2 -2
  7. data/lib/cli/progressbar.rb +10 -10
  8. data/lib/cli/tools.rb +3 -3
  9. data/lib/request_log_analyzer.rb +4 -4
  10. data/lib/request_log_analyzer/aggregator.rb +10 -10
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
  12. data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
  14. data/lib/request_log_analyzer/controller.rb +153 -69
  15. data/lib/request_log_analyzer/database.rb +13 -13
  16. data/lib/request_log_analyzer/database/base.rb +17 -17
  17. data/lib/request_log_analyzer/database/connection.rb +3 -3
  18. data/lib/request_log_analyzer/database/request.rb +2 -2
  19. data/lib/request_log_analyzer/database/source.rb +1 -1
  20. data/lib/request_log_analyzer/file_format.rb +15 -15
  21. data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
  22. data/lib/request_log_analyzer/file_format/apache.rb +20 -19
  23. data/lib/request_log_analyzer/file_format/merb.rb +12 -12
  24. data/lib/request_log_analyzer/file_format/rack.rb +4 -4
  25. data/lib/request_log_analyzer/file_format/rails.rb +146 -70
  26. data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
  27. data/lib/request_log_analyzer/filter.rb +6 -6
  28. data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
  29. data/lib/request_log_analyzer/filter/field.rb +9 -9
  30. data/lib/request_log_analyzer/filter/timespan.rb +12 -10
  31. data/lib/request_log_analyzer/line_definition.rb +15 -14
  32. data/lib/request_log_analyzer/log_processor.rb +22 -22
  33. data/lib/request_log_analyzer/mailer.rb +15 -9
  34. data/lib/request_log_analyzer/output.rb +53 -12
  35. data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
  36. data/lib/request_log_analyzer/output/html.rb +20 -20
  37. data/lib/request_log_analyzer/request.rb +35 -36
  38. data/lib/request_log_analyzer/source.rb +7 -7
  39. data/lib/request_log_analyzer/source/database_loader.rb +7 -7
  40. data/lib/request_log_analyzer/source/log_parser.rb +48 -43
  41. data/lib/request_log_analyzer/tracker.rb +128 -14
  42. data/lib/request_log_analyzer/tracker/duration.rb +39 -132
  43. data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
  44. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
  45. data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
  46. data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
  47. data/request-log-analyzer.gemspec +19 -15
  48. data/spec/fixtures/rails_22.log +1 -1
  49. data/spec/integration/command_line_usage_spec.rb +1 -1
  50. data/spec/lib/helpers.rb +7 -7
  51. data/spec/lib/macros.rb +3 -3
  52. data/spec/lib/matchers.rb +41 -27
  53. data/spec/lib/mocks.rb +15 -14
  54. data/spec/lib/testing_format.rb +9 -9
  55. data/spec/spec_helper.rb +6 -6
  56. data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
  57. data/spec/unit/aggregator/summarizer_spec.rb +4 -4
  58. data/spec/unit/controller/controller_spec.rb +2 -2
  59. data/spec/unit/controller/log_processor_spec.rb +1 -1
  60. data/spec/unit/database/base_class_spec.rb +19 -19
  61. data/spec/unit/database/connection_spec.rb +3 -3
  62. data/spec/unit/database/database_spec.rb +25 -25
  63. data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
  64. data/spec/unit/file_format/apache_format_spec.rb +13 -13
  65. data/spec/unit/file_format/file_format_api_spec.rb +13 -13
  66. data/spec/unit/file_format/line_definition_spec.rb +24 -17
  67. data/spec/unit/file_format/merb_format_spec.rb +41 -45
  68. data/spec/unit/file_format/rails_format_spec.rb +157 -117
  69. data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
  70. data/spec/unit/filter/field_filter_spec.rb +13 -13
  71. data/spec/unit/filter/filter_spec.rb +1 -1
  72. data/spec/unit/filter/timespan_filter_spec.rb +15 -15
  73. data/spec/unit/mailer_spec.rb +30 -0
  74. data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
  75. data/spec/unit/source/log_parser_spec.rb +27 -27
  76. data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
  77. data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
  78. data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
  79. data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
  80. data/spec/unit/tracker/tracker_api_spec.rb +13 -13
  81. data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
  82. data/tasks/github-gem.rake +125 -75
  83. data/tasks/request_log_analyzer.rake +2 -2
  84. metadata +8 -6
@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Anonymize.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should anonimize ip" do
10
10
  @filter.filter(request(:ip => '123.123.123.123'))[:ip].should_not eql('123.123.123.123')
11
11
  end
@@ -17,5 +17,5 @@ describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
17
17
  it "should fuzz durations" do
18
18
  @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
19
19
  end
20
-
20
+
21
21
  end
@@ -5,22 +5,22 @@ describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :select)
7
7
  end
8
-
8
+
9
9
  it "should reject a request if the field value does not match" do
10
10
  @filter.filter(request(:test => 'not test')).should be_nil
11
11
  end
12
-
12
+
13
13
  it "should reject a request if the field name does not match" do
14
14
  @filter.filter(request(:testing => 'test')).should be_nil
15
15
  end
16
16
 
17
17
  it "should accept a request if the both name and value match" do
18
18
  @filter.filter(request(:test => 'test')).should_not be_nil
19
- end
20
-
19
+ end
20
+
21
21
  it "should accept a request if the value is not the first value" do
22
22
  @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
23
- end
23
+ end
24
24
  end
25
25
 
26
26
  describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
@@ -28,22 +28,22 @@ describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
28
28
  before(:each) do
29
29
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :reject)
30
30
  end
31
-
31
+
32
32
  it "should accept a request if the field value does not match" do
33
33
  @filter.filter(request(:test => 'not test')).should_not be_nil
34
34
  end
35
-
35
+
36
36
  it "should accept a request if the field name does not match" do
37
37
  @filter.filter(request(:testing => 'test')).should_not be_nil
38
38
  end
39
39
 
40
40
  it "should reject a request if the both name and value match" do
41
41
  @filter.filter(request(:test => 'test')).should be_nil
42
- end
43
-
42
+ end
43
+
44
44
  it "should reject a request if the value is not the first value" do
45
45
  @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
46
- end
46
+ end
47
47
  end
48
48
 
49
49
  describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
@@ -51,16 +51,16 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
51
51
  before(:each) do
52
52
  @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => '/test/', :mode => :select)
53
53
  end
54
-
54
+
55
55
  it "should reject a request if the field value does not match" do
56
56
  @filter.filter(request(:test => 'a working test')).should_not be_nil
57
57
  end
58
-
58
+
59
59
  it "should reject a request if the field name does not match" do
60
60
  @filter.filter(request(:testing => 'test')).should be_nil
61
61
  end
62
62
 
63
63
  it "should accept a request if the value is not the first value" do
64
64
  @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
65
- end
65
+ end
66
66
  end
@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::Filter::Base, 'base filter' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Base.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should return everything" do
10
10
  @filter.filter(request(:ip => '123.123.123.123'))[:ip].should eql('123.123.123.123')
11
11
  end
@@ -5,54 +5,54 @@ describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
5
5
  before(:each) do
6
6
  @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
7
7
  end
8
-
8
+
9
9
  it "should reject a request before the after date" do
10
10
  @filter.filter(request(:timestamp => 20081212000000)).should be_nil
11
11
  end
12
-
12
+
13
13
  it "should reject a request after the before date" do
14
14
  @filter.filter(request(:timestamp => 20090303000000)).should be_nil
15
15
  end
16
-
16
+
17
17
  it "should accept a request between the after and before dates" do
18
18
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
19
19
  end
20
20
  end
21
21
 
22
22
  describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
23
-
23
+
24
24
  before(:each) do
25
- @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :before => DateTime.parse('2009-02-02'))
25
+ @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :before => DateTime.parse('2009-02-02'))
26
26
  end
27
-
27
+
28
28
  it "should accept a request before the after date" do
29
29
  @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
30
30
  end
31
-
31
+
32
32
  it "should reject a request after the before date" do
33
33
  @filter.filter(request(:timestamp => 20090303000000)).should be_nil
34
34
  end
35
-
35
+
36
36
  it "should accept a request between the after and before dates" do
37
37
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
38
- end
38
+ end
39
39
  end
40
40
 
41
- describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
42
-
41
+ describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
42
+
43
43
  before(:each) do
44
44
  @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'))
45
45
  end
46
-
46
+
47
47
  it "should reject a request before the after date" do
48
48
  @filter.filter(request(:timestamp => 20081212000000)).should be_nil
49
49
  end
50
-
50
+
51
51
  it "should accept a request after the before date" do
52
52
  @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
53
53
  end
54
-
54
+
55
55
  it "should accept a request between the after and before dates" do
56
56
  @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
57
- end
57
+ end
58
58
  end
@@ -0,0 +1,30 @@
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
+
3
+ describe RequestLogAnalyzer::Mailer, 'mailer' do
4
+
5
+ it "should store printed data" do
6
+ @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', :debug => true)
7
+
8
+ @mailer << 'test1'
9
+ @mailer.puts 'test2'
10
+
11
+ @mailer.data.should eql(['test1', 'test2'])
12
+ end
13
+
14
+ it "should send mail" do
15
+ @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', :debug => true)
16
+
17
+ @mailer << 'test1'
18
+ @mailer.puts 'test2'
19
+
20
+ mail = @mailer.mail
21
+
22
+ mail[0].should include("contact@railsdoctors.com")
23
+ mail[0].should include("test1")
24
+ mail[0].should include("test2")
25
+
26
+ mail[1].should include("contact@railsdoctors.com")
27
+ mail[2].should include("alfa@beta.com")
28
+ end
29
+
30
+ end
@@ -1,49 +1,49 @@
1
- require File.dirname(__FILE__) + '/../../spec_helper'
1
+ require File.dirname(__FILE__) + '/../spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Request do
4
-
4
+
5
5
  before(:each) do
6
6
  @request = testing_format.request
7
- end
8
-
7
+ end
8
+
9
9
  it "should be empty without any captured lines in it" do
10
10
  @request.should be_empty
11
11
  end
12
-
12
+
13
13
  context :incomplete do
14
-
14
+
15
15
  before(:each) do
16
16
  @request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
17
17
  end
18
-
18
+
19
19
  it "should be single if only one line has been added" do
20
20
  @request.should_not be_empty
21
21
  end
22
-
22
+
23
23
  it "should not be a completed request" do
24
24
  @request.should_not be_completed
25
- end
26
-
25
+ end
26
+
27
27
  it "should take the line type of the first line as global line_type" do
28
28
  @request.lines[0][:line_type].should == :test
29
29
  @request.should =~ :test
30
30
  end
31
-
31
+
32
32
  it "should return the first field value" do
33
33
  @request[:test_capture].should == 'awesome!'
34
34
  end
35
-
35
+
36
36
  it "should return nil if no such field is present" do
37
37
  @request[:nonexisting].should be_nil
38
38
  end
39
39
  end
40
-
40
+
41
41
  context :completed do
42
-
42
+
43
43
  before(:each) do
44
- @request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
45
- @request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
46
- @request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
44
+ @request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
45
+ @request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
46
+ @request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
47
47
  @request << { :line_type => :last, :lineno => 10, :time => 0.03 }
48
48
  end
49
49
 
@@ -64,29 +64,29 @@ describe RequestLogAnalyzer::Request do
64
64
  @request[:time].should == 0.03
65
65
  end
66
66
 
67
- it "should detect the first matching field value" do
67
+ it "should detect the first matching field value" do
68
68
  @request.first(:test_capture).should == 'testing'
69
69
  end
70
70
 
71
- it "should detect the every matching field value" do
71
+ it "should detect the every matching field value" do
72
72
  @request.every(:test_capture).should == ['testing', "testing some more"]
73
73
  end
74
74
 
75
75
  it "should set the first_lineno for a request to the lowest lineno encountered" do
76
76
  @request.first_lineno.should eql(1)
77
77
  end
78
-
78
+
79
79
  it "should set the first_lineno for a request if a line with a lower lineno is added" do
80
- @request << { :line_type => :test, :lineno => 0 }
80
+ @request << { :line_type => :test, :lineno => 0 }
81
81
  @request.first_lineno.should eql(0)
82
- end
82
+ end
83
83
 
84
84
  it "should set the last_lineno for a request to the highest encountered lineno" do
85
85
  @request.last_lineno.should eql(10)
86
86
  end
87
-
87
+
88
88
  it "should not set the last_lineno for a request if a line with a lower lineno is added" do
89
- @request << { :line_type => :test, :lineno => 7 }
89
+ @request << { :line_type => :test, :lineno => 7 }
90
90
  @request.last_lineno.should eql(10)
91
91
  end
92
92
 
@@ -95,17 +95,17 @@ describe RequestLogAnalyzer::Request do
95
95
  end
96
96
 
97
97
  it "should set return a timestamp field if such a field is captured" do
98
- @request << { :line_type => :first, :lineno => 1, :name => 'first line!', :timestamp => Time.now}
98
+ @request << { :line_type => :first, :lineno => 1, :name => 'first line!', :timestamp => Time.now}
99
99
  @request.timestamp.should_not be_nil
100
- end
100
+ end
101
101
  end
102
-
102
+
103
103
  context 'single line' do
104
104
  # combined is both a header and a footer line
105
105
  before(:each) { @request << { :line_type => :combined, :lineno => 1 } }
106
-
106
+
107
107
  it "should be a completed request if the line is both header and footer" do
108
108
  @request.should be_completed
109
- end
109
+ end
110
110
  end
111
- end
111
+ end
@@ -1,60 +1,60 @@
1
1
  require File.dirname(__FILE__) + '/../../spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::Source::LogParser, :requests do
4
-
4
+
5
5
  before(:each) do
6
6
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
7
7
  end
8
-
8
+
9
9
  it "should have multiple line definitions" do
10
10
  @log_parser.file_format.line_definitions.length.should >= 2
11
- end
12
-
11
+ end
12
+
13
13
  it "should have a valid language" do
14
14
  @log_parser.file_format.should be_valid
15
15
  end
16
-
16
+
17
17
  it "should set the :source for every parsed line" do
18
18
  @log_parser.parse_file(log_fixture(:rails_22)) do |request|
19
19
  request.lines.all? { |line| line[:source] == log_fixture(:rails_22) }.should be_true
20
20
  end
21
21
  end
22
-
22
+
23
23
  it "should set the :lineno for every parsed line" do
24
24
  @log_parser.parse_file(log_fixture(:rails_22)) do |request|
25
25
  request.lines.all? { |line| line.has_key?(:lineno) }.should be_true
26
26
  end
27
27
  end
28
-
28
+
29
29
  it "should parse more lines than requests" do
30
30
  @log_parser.should_receive(:handle_request).with(an_instance_of(TestingFormat::Request)).twice
31
31
  @log_parser.parse_file(log_fixture(:test_language_combined))
32
- @log_parser.parsed_lines.should > 2
32
+ @log_parser.parsed_lines.should > 2
33
33
  end
34
-
34
+
35
35
  it "should parse requests spanned over multiple files" do
36
36
  @log_parser.should_receive(:handle_request).with(an_instance_of(TestingFormat::Request)).once
37
37
  @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
38
38
  end
39
-
39
+
40
40
  it "should parse all request values when spanned over multiple files" do
41
41
  @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
42
- request.lines.should have(4).items
42
+ request.lines.should have(4).items
43
43
  request[:request_no].should == 1
44
- request[:test_capture].should == "Testing is amazing" # Note the custom converter
44
+ request[:test_capture].should == "Testing is amazing" # Note the custom converter
45
45
  end
46
- end
47
-
46
+ end
47
+
48
48
  it "should parse a stream and find valid requests" do
49
49
  io = File.new(log_fixture(:test_file_format), 'r')
50
50
  @log_parser.parse_io(io) do |request|
51
51
  request.should be_kind_of(RequestLogAnalyzer::Request)
52
52
  request.should =~ :test
53
- request[:test_capture].should_not be_nil
53
+ request[:test_capture].should_not be_nil
54
54
  end
55
55
  io.close
56
56
  end
57
-
57
+
58
58
  it "should parse a request that only consists of one line" do
59
59
  @log_parser.parse_file(log_fixture(:header_and_footer))
60
60
  @log_parser.parsed_requests.should == 2
@@ -62,30 +62,30 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
62
62
  end
63
63
 
64
64
  describe RequestLogAnalyzer::Source::LogParser, :warnings do
65
-
65
+
66
66
  before(:each) do
67
67
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format, :parse_strategy => 'cautious')
68
68
  end
69
-
69
+
70
70
  it "should warn about teaser matching problems" do
71
71
  @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
72
72
  @log_parser.parse_file(log_fixture(:test_file_format))
73
73
  end
74
-
74
+
75
75
  it "should warn about unmatching request headers and footers" do
76
- @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
77
- @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
78
- @log_parser.should_not_receive(:handle_request)
79
- @log_parser.parse_file(log_fixture(:test_order))
80
- end
76
+ @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
77
+ @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
78
+ @log_parser.should_not_receive(:handle_request)
79
+ @log_parser.parse_file(log_fixture(:test_order))
80
+ end
81
81
  end
82
82
 
83
83
  describe RequestLogAnalyzer::Source::LogParser, :decompression do
84
-
84
+
85
85
  before(:each) do
86
86
  @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat::Rails.create)
87
87
  end
88
-
88
+
89
89
  it "should parse a rails gzipped log file" do
90
90
  @log_parser.should_receive(:handle_request).once
91
91
  @log_parser.parse_file(log_fixture(:decompression, "log.gz"))
@@ -115,5 +115,5 @@ describe RequestLogAnalyzer::Source::LogParser, :decompression do
115
115
  @log_parser.parse_file(log_fixture(:decompression, "log.zip"))
116
116
  @log_parser.parsed_lines.should > 0
117
117
  end
118
-
118
+
119
119
  end