request-log-analyzer 1.0.2 → 1.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. data/.gitignore +10 -0
  2. data/DESIGN.rdoc +41 -0
  3. data/LICENSE +4 -4
  4. data/README.rdoc +38 -0
  5. data/Rakefile +6 -3
  6. data/bin/request-log-analyzer +70 -72
  7. data/lib/cli/command_line_arguments.rb +53 -53
  8. data/lib/cli/database_console.rb +26 -0
  9. data/lib/cli/database_console_init.rb +43 -0
  10. data/lib/cli/progressbar.rb +166 -189
  11. data/lib/cli/tools.rb +49 -0
  12. data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
  15. data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
  16. data/lib/request_log_analyzer/controller.rb +251 -98
  17. data/lib/request_log_analyzer/database/base.rb +114 -0
  18. data/lib/request_log_analyzer/database/connection.rb +38 -0
  19. data/lib/request_log_analyzer/database/request.rb +22 -0
  20. data/lib/request_log_analyzer/database/source.rb +13 -0
  21. data/lib/request_log_analyzer/database/warning.rb +14 -0
  22. data/lib/request_log_analyzer/database.rb +102 -0
  23. data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
  24. data/lib/request_log_analyzer/file_format/apache.rb +147 -0
  25. data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
  26. data/lib/request_log_analyzer/file_format/merb.rb +65 -29
  27. data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
  28. data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
  29. data/lib/request_log_analyzer/file_format/rack.rb +9 -0
  30. data/lib/request_log_analyzer/file_format/rails.rb +164 -78
  31. data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
  32. data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
  33. data/lib/request_log_analyzer/file_format.rb +252 -58
  34. data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
  35. data/lib/request_log_analyzer/filter/field.rb +19 -13
  36. data/lib/request_log_analyzer/filter/timespan.rb +25 -12
  37. data/lib/request_log_analyzer/filter.rb +30 -0
  38. data/lib/request_log_analyzer/line_definition.rb +69 -96
  39. data/lib/request_log_analyzer/log_processor.rb +31 -53
  40. data/lib/request_log_analyzer/mailer.rb +65 -0
  41. data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
  42. data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
  43. data/lib/request_log_analyzer/output/html.rb +187 -0
  44. data/lib/request_log_analyzer/output.rb +117 -0
  45. data/lib/request_log_analyzer/request.rb +125 -40
  46. data/lib/request_log_analyzer/source/database_loader.rb +87 -0
  47. data/lib/request_log_analyzer/source/log_parser.rb +297 -0
  48. data/lib/request_log_analyzer/source.rb +72 -0
  49. data/lib/request_log_analyzer/tracker/duration.rb +28 -64
  50. data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
  51. data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
  52. data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
  53. data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
  54. data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
  55. data/lib/request_log_analyzer/tracker.rb +101 -0
  56. data/lib/request_log_analyzer.rb +43 -13
  57. data/request-log-analyzer.gemspec +41 -0
  58. data/spec/database.yml +23 -0
  59. data/spec/fixtures/apache_combined.log +5 -0
  60. data/spec/fixtures/apache_common.log +10 -0
  61. data/spec/fixtures/decompression.log +12 -0
  62. data/spec/fixtures/decompression.log.bz2 +0 -0
  63. data/spec/fixtures/decompression.log.gz +0 -0
  64. data/spec/fixtures/decompression.log.zip +0 -0
  65. data/spec/fixtures/decompression.tar.gz +0 -0
  66. data/spec/fixtures/decompression.tgz +0 -0
  67. data/spec/fixtures/header_and_footer.log +6 -0
  68. data/spec/fixtures/merb_prefixed.log +9 -0
  69. data/spec/fixtures/mysql_slow_query.log +110 -0
  70. data/spec/fixtures/postgresql.log +2980 -0
  71. data/spec/fixtures/rails.db +0 -0
  72. data/spec/fixtures/rails_22.log +1 -1
  73. data/spec/fixtures/sinatra.log +99 -0
  74. data/spec/integration/command_line_usage_spec.rb +84 -0
  75. data/spec/integration/mailer_spec.rb +179 -0
  76. data/spec/integration/munin_plugins_rails_spec.rb +58 -0
  77. data/spec/integration/scout_spec.rb +152 -0
  78. data/spec/lib/helpers.rb +72 -0
  79. data/spec/lib/macros.rb +18 -0
  80. data/spec/lib/matchers.rb +77 -0
  81. data/spec/lib/mocks.rb +77 -0
  82. data/spec/lib/testing_format.rb +46 -0
  83. data/spec/spec_helper.rb +16 -59
  84. data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
  85. data/spec/unit/aggregator/summarizer_spec.rb +26 -0
  86. data/spec/unit/controller/controller_spec.rb +41 -0
  87. data/spec/unit/controller/log_processor_spec.rb +18 -0
  88. data/spec/unit/database/base_class_spec.rb +183 -0
  89. data/spec/unit/database/connection_spec.rb +34 -0
  90. data/spec/unit/database/database_spec.rb +133 -0
  91. data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
  92. data/spec/unit/file_format/apache_format_spec.rb +203 -0
  93. data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
  94. data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
  95. data/spec/unit/file_format/file_format_api_spec.rb +69 -0
  96. data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
  97. data/spec/unit/file_format/line_definition_spec.rb +75 -0
  98. data/spec/unit/file_format/merb_format_spec.rb +52 -0
  99. data/spec/unit/file_format/mysql_format_spec.rb +154 -0
  100. data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
  101. data/spec/unit/file_format/rack_format_spec.rb +50 -0
  102. data/spec/unit/file_format/rails3_format_spec.rb +120 -0
  103. data/spec/unit/file_format/rails_format_spec.rb +180 -0
  104. data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
  105. data/spec/unit/filter/field_filter_spec.rb +66 -0
  106. data/spec/unit/filter/filter_spec.rb +17 -0
  107. data/spec/unit/filter/timespan_filter_spec.rb +58 -0
  108. data/spec/unit/mailer_spec.rb +42 -0
  109. data/spec/unit/request_spec.rb +111 -0
  110. data/spec/unit/source/log_parser_spec.rb +119 -0
  111. data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
  112. data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
  113. data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
  114. data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
  115. data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
  116. data/spec/unit/tracker/tracker_api_spec.rb +125 -0
  117. data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
  118. data/tasks/github-gem.rake +290 -139
  119. data/tasks/request_log_analyzer.rake +23 -7
  120. metadata +221 -94
  121. data/DESIGN +0 -14
  122. data/HACKING +0 -7
  123. data/README.textile +0 -36
  124. data/lib/cli/bashcolorizer.rb +0 -60
  125. data/lib/request_log_analyzer/aggregator/database.rb +0 -148
  126. data/lib/request_log_analyzer/filter/base.rb +0 -29
  127. data/lib/request_log_analyzer/log_parser.rb +0 -173
  128. data/lib/request_log_analyzer/source/base.rb +0 -42
  129. data/lib/request_log_analyzer/source/log_file.rb +0 -170
  130. data/lib/request_log_analyzer/tracker/base.rb +0 -54
  131. data/lib/request_log_analyzer/tracker/category.rb +0 -71
  132. data/spec/controller_spec.rb +0 -40
  133. data/spec/database_inserter_spec.rb +0 -101
  134. data/spec/file_format_spec.rb +0 -78
  135. data/spec/file_formats/spec_format.rb +0 -26
  136. data/spec/filter_spec.rb +0 -137
  137. data/spec/line_definition_spec.rb +0 -124
  138. data/spec/log_parser_spec.rb +0 -68
  139. data/spec/log_processor_spec.rb +0 -57
  140. data/spec/merb_format_spec.rb +0 -38
  141. data/spec/rails_format_spec.rb +0 -76
  142. data/spec/request_spec.rb +0 -72
  143. data/spec/summarizer_spec.rb +0 -9
  144. data/tasks/rspec.rake +0 -6
@@ -1,78 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::FileFormat, :format_definition do
4
-
5
- before(:each) do
6
- @first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
7
- @second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
8
- end
9
-
10
- it "should specify lines with a hash" do
11
-
12
- @first_file_format.new.should have(0).line_definitions
13
-
14
- @first_file_format.format_definition do |line|
15
- line.hash_test :regexp => /test/, :captures => []
16
- end
17
-
18
- @format_instance = @first_file_format.new
19
- @format_instance.should have(1).line_definitions
20
- @format_instance.line_definitions[:hash_test].should_not be_nil
21
- end
22
-
23
- it "should specift lines directly" do
24
- @first_file_format.new.should have(0).line_definitions
25
-
26
- @first_file_format.format_definition.direct_test do |line|
27
- line.regexp = /test/
28
- end
29
-
30
- @first_file_format.new.line_definitions[:direct_test].should_not be_nil
31
- end
32
-
33
- it "specify lines with a block" do
34
-
35
- @first_file_format.new.should have(0).line_definitions
36
-
37
- @first_file_format.format_definition do |format|
38
- format.block_test do |line|
39
- line.regexp = /test/
40
- line.captures = []
41
- end
42
- end
43
-
44
- @first_file_format.new.line_definitions[:block_test].should_not be_nil
45
- end
46
-
47
- it "should define lines only for itself" do
48
-
49
- @first_file_format.format_definition do |line|
50
- line.first_test :regexp => /test/, :captures => []
51
- end
52
-
53
- @second_file_format.format_definition do |line|
54
- line.second_test :regexp => /test/, :captures => []
55
- end
56
-
57
- @first_file_format.new.should have(1).line_definitions
58
- @first_file_format.new.line_definitions[:first_test].should_not be_nil
59
- @second_file_format.new.should have(1).line_definitions
60
- @second_file_format.new.line_definitions[:second_test].should_not be_nil
61
- end
62
- end
63
-
64
- describe RequestLogAnalyzer::FileFormat, :load do
65
-
66
- include RequestLogAnalyzerSpecHelper
67
-
68
- it "should load a predefined file format from the /file_format dir" do
69
- @file_format = RequestLogAnalyzer::FileFormat.load(:rails)
70
- @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
71
- end
72
-
73
- it "should load a provided format file" do
74
- @file_format = RequestLogAnalyzer::FileFormat.load(format_file(:spec_format))
75
- @file_format.should be_kind_of(SpecFormat)
76
- end
77
-
78
- end
@@ -1,26 +0,0 @@
1
- class SpecFormat < RequestLogAnalyzer::FileFormat
2
-
3
- format_definition.first do |line|
4
- line.header = true
5
- line.teaser = /processing /
6
- line.regexp = /processing request (\d+)/
7
- line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]
8
- end
9
-
10
- format_definition.test do |line|
11
- line.teaser = /testing /
12
- line.regexp = /testing is (\w+)/
13
- line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
14
- end
15
-
16
- format_definition.last do |line|
17
- line.footer = true
18
- line.teaser = /finishing /
19
- line.regexp = /finishing request (\d+)/
20
- line.captures = [{ :name => :request_no, :type => :integer}]
21
- end
22
-
23
- report do |analyze|
24
- analyze.category :test_capture, :title => 'What is testing exactly?'
25
- end
26
- end
data/spec/filter_spec.rb DELETED
@@ -1,137 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
- require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
3
- require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
4
-
5
- describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
6
- include RequestLogAnalyzerSpecHelper
7
-
8
- before(:each) do
9
- @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
10
- @filter.prepare
11
- end
12
-
13
- it "should reject a request before the after date" do
14
- @filter.filter(request(:timestamp => 20081212000000)).should be_nil
15
- end
16
-
17
- it "should reject a request after the before date" do
18
- @filter.filter(request(:timestamp => 20090303000000)).should be_nil
19
- end
20
-
21
- it "should accept a request between the after and before dates" do
22
- @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
23
- end
24
- end
25
-
26
- describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
27
- include RequestLogAnalyzerSpecHelper
28
-
29
- before(:each) do
30
- @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))
31
- @filter.prepare
32
- end
33
-
34
- it "should accept a request before the after date" do
35
- @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
36
- end
37
-
38
- it "should reject a request after the before date" do
39
- @filter.filter(request(:timestamp => 20090303000000)).should be_nil
40
- end
41
-
42
- it "should accept a request between the after and before dates" do
43
- @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
44
- end
45
- end
46
-
47
- describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
48
- include RequestLogAnalyzerSpecHelper
49
-
50
- before(:each) do
51
- @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
52
- @filter.prepare
53
- end
54
-
55
- it "should reject a request before the after date" do
56
- @filter.filter(request(:timestamp => 20081212000000)).should be_nil
57
- end
58
-
59
- it "should accept a request after the before date" do
60
- @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
61
- end
62
-
63
- it "should accept a request between the after and before dates" do
64
- @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
65
- end
66
- end
67
-
68
- describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
69
- include RequestLogAnalyzerSpecHelper
70
-
71
- before(:each) do
72
- @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
73
- @filter.prepare
74
- end
75
-
76
- it "should reject a request if the field value does not match" do
77
- @filter.filter(request(:test => 'not test')).should be_nil
78
- end
79
-
80
- it "should reject a request if the field name does not match" do
81
- @filter.filter(request(:testing => 'test')).should be_nil
82
- end
83
-
84
- it "should accept a request if the both name and value match" do
85
- @filter.filter(request(:test => 'test')).should_not be_nil
86
- end
87
-
88
- it "should accept a request if the value is not the first value" do
89
- @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
90
- end
91
- end
92
-
93
- describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
94
- include RequestLogAnalyzerSpecHelper
95
-
96
- before(:each) do
97
- @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
98
- @filter.prepare
99
- end
100
-
101
- it "should accept a request if the field value does not match" do
102
- @filter.filter(request(:test => 'not test')).should_not be_nil
103
- end
104
-
105
- it "should accept a request if the field name does not match" do
106
- @filter.filter(request(:testing => 'test')).should_not be_nil
107
- end
108
-
109
- it "should reject a request if the both name and value match" do
110
- @filter.filter(request(:test => 'test')).should be_nil
111
- end
112
-
113
- it "should reject a request if the value is not the first value" do
114
- @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
115
- end
116
- end
117
-
118
- describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
119
- include RequestLogAnalyzerSpecHelper
120
-
121
- before(:each) do
122
- @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
123
- @filter.prepare
124
- end
125
-
126
- it "should reject a request if the field value does not match" do
127
- @filter.filter(request(:test => 'a working test')).should_not be_nil
128
- end
129
-
130
- it "should reject a request if the field name does not match" do
131
- @filter.filter(request(:testing => 'test')).should be_nil
132
- end
133
-
134
- it "should accept a request if the value is not the first value" do
135
- @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
136
- end
137
- end
@@ -1,124 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::LineDefinition, :parsing do
4
-
5
- before(:each) do
6
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
7
- :teaser => /Testing /,
8
- :regexp => /Testing (\w+), tries\: (\d+)/,
9
- :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
10
- })
11
- end
12
-
13
- it "should return false on an unmatching line" do
14
- (@line_definition =~ "nonmatching").should be_false
15
- end
16
-
17
- it "should return false when only the teaser matches" do
18
- (@line_definition =~ "Testing LineDefinition").should be_false
19
- end
20
-
21
- it "should return a hash if the line matches" do
22
- (@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
23
- end
24
-
25
- it "should return a hash with all captures set" do
26
- hash = @line_definition.matches("Testing LineDefinition, tries: 123")
27
- hash[:what].should == "LineDefinition"
28
- hash[:tries].should == 123
29
- end
30
-
31
- it "should return a hash with :line_type set" do
32
- @line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
33
- end
34
- end
35
-
36
- describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
37
- before(:each) do
38
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
39
- :teaser => /Anonymize /,
40
- :regexp => /Anonymize (\w+)!/,
41
- :captures => [{ :name => :what, :type => :string }]
42
- })
43
- end
44
-
45
- it "should return nil if the teaser does not match" do
46
- @line_definition.anonymize("Nonsense").should be_nil
47
- end
48
-
49
- it "should return nil if no teaser exists and the regexp doesn't match" do
50
- line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
51
- :regexp => /Anonymize!/, :captures => []})
52
-
53
- line_definition.anonymize('nonsense').should be_nil
54
- end
55
-
56
- it "should return itself if only the teaser matches" do
57
- @line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
58
- end
59
-
60
- it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
61
- @line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
62
- end
63
-
64
- it "should return a string if the line matches" do
65
- @line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
66
- end
67
-
68
- it "should not anonymize :what" do
69
- @line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
70
- end
71
- end
72
-
73
- describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
74
-
75
- it "should anonymize completely if anonymize is true" do
76
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
77
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
78
-
79
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
80
- end
81
-
82
- it "should anonymize a URL" do
83
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
84
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
85
-
86
- @line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
87
- end
88
-
89
- it "should anonymize an IP address" do
90
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
91
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
92
-
93
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
94
- end
95
-
96
- it "should anonymize completely if the anonymizer is unknown" do
97
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
98
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
99
-
100
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
101
- end
102
-
103
- it "should anonymize an integer slightly" do
104
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
105
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
106
-
107
- @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
108
- end
109
-
110
- it "should anonymize an integer slightly" do
111
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
112
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
113
-
114
- @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
115
- end
116
-
117
- it "should anonymize an double slightly" do
118
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
119
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
120
-
121
- @line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
122
- end
123
-
124
- end
@@ -1,68 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::LogParser, :requests do
4
- include RequestLogAnalyzerSpecHelper
5
-
6
- before(:each) do
7
- @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
8
- end
9
-
10
- it "should have multiple line definitions" do
11
- @log_parser.file_format.line_definitions.length.should >= 2
12
- end
13
-
14
- it "should have a valid language" do
15
- @log_parser.file_format.should be_valid
16
- end
17
-
18
- it "should parse more lines than requests" do
19
- @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
20
- @log_parser.parse_file(log_fixture(:test_language_combined))
21
- @log_parser.parsed_lines.should > 2
22
- end
23
-
24
- it "should parse requests spanned over multiple files" do
25
- @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).once
26
- @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
27
- end
28
-
29
- it "should parse all request values when spanned over multiple files" do
30
- @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
31
- request.lines.should have(4).items
32
-
33
- request[:request_no].should == 1
34
- request[:test_capture].should == "amazing"
35
- end
36
- end
37
-
38
- it "should parse a stream and find valid requests" do
39
- io = File.new(log_fixture(:test_file_format), 'r')
40
- @log_parser.parse_io(io) do |request|
41
- request.should be_kind_of(RequestLogAnalyzer::Request)
42
- request.should =~ :test
43
- request[:test_capture].should_not be_nil
44
- end
45
- io.close
46
- end
47
-
48
- end
49
-
50
- describe RequestLogAnalyzer::LogParser, :warnings do
51
- include RequestLogAnalyzerSpecHelper
52
-
53
- before(:each) do
54
- @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
55
- end
56
-
57
- it "should warn about teaser matching problems" do
58
- @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
59
- @log_parser.parse_file(log_fixture(:test_file_format))
60
- end
61
-
62
- it "should warn about unmatching request headers and footers" do
63
- @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
64
- @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
65
- @log_parser.should_not_receive(:handle_request)
66
- @log_parser.parse_file(log_fixture(:test_order))
67
- end
68
- end
@@ -1,57 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
- require 'request_log_analyzer/log_processor'
3
-
4
- describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
5
-
6
- include RequestLogAnalyzerSpecHelper
7
-
8
- before(:each) do
9
- @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
10
- @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
11
- end
12
-
13
- it "should keep a junk line if :keep_junk_lines is true" do
14
- @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
15
- end
16
-
17
- it "should remove a junk line" do
18
- @log_anonymizer.anonymize_line("junk line\n").should be_empty
19
- end
20
-
21
- it "should keep a teaser line intact" do
22
- @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
23
- end
24
-
25
- it "should discard a teaser line if discard_teaser_line is true" do
26
- @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
27
- end
28
-
29
- it "should keep a matching line intact if no anonymizing is declared" do
30
- @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
31
- end
32
-
33
- it "should anonymize values completely if requested" do
34
- @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
35
- end
36
-
37
- it "should anonymize values slightly if requested" do
38
- @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
39
- end
40
- end
41
-
42
- describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
43
-
44
- include RequestLogAnalyzerSpecHelper
45
-
46
- before(:each) do
47
- @log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
48
- end
49
-
50
- it "should remove a junk line" do
51
- @log_stripper.strip_line("junk line\n").should be_empty
52
- end
53
-
54
- it "should keep a teaser line intact" do
55
- @log_stripper.strip_line("processing 1234\n").should be_empty
56
- end
57
- end
@@ -1,38 +0,0 @@
1
- # require File.dirname(__FILE__) + '/spec_helper'
2
- #
3
- # describe RequestLogAnalyzer::LogParser, "Merb" do
4
- # include RequestLogAnalyzerSpecHelper
5
- #
6
- # before(:each) do
7
- # @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
8
- # end
9
- #
10
- # it "should have a valid language definitions" do
11
- # @log_parser.file_format.should be_valid
12
- # end
13
- #
14
- # it "should parse a stream and find valid requests" do
15
- # File.open(log_fixture(:merb), 'r') do |io|
16
- # @log_parser.parse_io(io) do |request|
17
- # request.should be_kind_of(RequestLogAnalyzer::Request)
18
- # end
19
- # end
20
- # end
21
- #
22
- # it "should find 11 completed requests" do
23
- # @log_parser.should_receive(:handle_request).exactly(11).times
24
- # @log_parser.parse_file(log_fixture(:merb))
25
- # end
26
- #
27
- # it "should parse all details from a request correctly" do
28
- # request = nil
29
- # @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
30
- #
31
- # request.should be_completed
32
- # request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008')
33
- # request[:dispatch_time].should == 0.243424
34
- # request[:after_filters_time].should == 6.9e-05
35
- # request[:before_filters_time].should == 0.213213
36
- # request[:action_time].should == 0.241652
37
- # end
38
- # end
@@ -1,76 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::LogParser, "Rails" do
4
- include RequestLogAnalyzerSpecHelper
5
-
6
- before(:each) do
7
- @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
8
- end
9
-
10
- it "should have a valid language definitions" do
11
- @log_parser.file_format.should be_valid
12
- end
13
-
14
- it "should parse a stream and find valid requests" do
15
- io = File.new(log_fixture(:rails_1x), 'r')
16
- @log_parser.parse_io(io) do |request|
17
- request.should be_kind_of(RequestLogAnalyzer::Request)
18
- end
19
- io.close
20
- end
21
-
22
- it "should find 4 completed requests" do
23
- @log_parser.should_not_receive(:warn)
24
- @log_parser.should_receive(:handle_request).exactly(4).times
25
- @log_parser.parse_file(log_fixture(:rails_1x))
26
- end
27
-
28
- it "should parse a Rails 2.2 request properly" do
29
- @log_parser.should_not_receive(:warn)
30
- @log_parser.parse_file(log_fixture(:rails_22)) do |request|
31
- request.should =~ :processing
32
- request.should =~ :completed
33
-
34
- request[:controller].should == 'PageController'
35
- request[:action].should == 'demo'
36
- request[:url].should == 'http://www.example.coml/demo'
37
- request[:status].should == 200
38
- request[:duration].should == 0.614
39
- request[:db].should == 0.031
40
- request[:view].should == 0.120
41
- end
42
- end
43
-
44
- it "should parse a syslog file with prefix correctly" do
45
- @log_parser.should_not_receive(:warn)
46
- @log_parser.parse_file(log_fixture(:syslog_1x)) do |request|
47
-
48
- request.should be_completed
49
-
50
- request[:controller].should == 'EmployeeController'
51
- request[:action].should == 'index'
52
- request[:url].should == 'http://example.com/employee.xml'
53
- request[:status].should == 200
54
- request[:duration].should == 0.21665
55
- request[:db].should == 0.0
56
- request[:view].should == 0.00926
57
- end
58
- end
59
-
60
- it "should parse cached requests" do
61
- @log_parser.should_not_receive(:warn)
62
- @log_parser.parse_file(log_fixture(:rails_22_cached)) do |request|
63
- request.should be_completed
64
- request =~ :cache_hit
65
- end
66
- end
67
-
68
- it "should detect unordered requests in the logs" do
69
- @log_parser.should_not_receive(:handle_request)
70
- # the first Processing-line will not give a warning, but the next one will
71
- @log_parser.should_receive(:warn).with(:unclosed_request, anything).once
72
- # Both Completed ;ines will give a warning
73
- @log_parser.should_receive(:warn).with(:no_current_request, anything).twice
74
- @log_parser.parse_file(log_fixture(:rails_unordered))
75
- end
76
- end
data/spec/request_spec.rb DELETED
@@ -1,72 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::Request, :incomplete_request do
4
-
5
- include RequestLogAnalyzerSpecHelper
6
-
7
- before(:each) do
8
- @incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
9
- @incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
10
- end
11
-
12
- it "should be single if only one line has been added" do
13
- @incomplete_request.should_not be_empty
14
- end
15
-
16
- it "should not be a completed request" do
17
- @incomplete_request.should_not be_completed
18
- end
19
-
20
- it "should take the line type of the first line as global line_type" do
21
- @incomplete_request.lines[0][:line_type].should == :test
22
- @incomplete_request.should =~ :test
23
- end
24
-
25
- it "should return the first field value" do
26
- @incomplete_request[:test_capture].should == 'awesome!'
27
- end
28
-
29
- it "should return nil if no such field is present" do
30
- @incomplete_request[:nonexisting].should be_nil
31
- end
32
- end
33
-
34
-
35
- describe RequestLogAnalyzer::Request, :completed_request do
36
-
37
- include RequestLogAnalyzerSpecHelper
38
-
39
- before(:each) do
40
- @completed_request = RequestLogAnalyzer::Request.new(spec_format)
41
- @completed_request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
42
- @completed_request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
43
- @completed_request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
44
- @completed_request << { :line_type => :last, :lineno => 10, :time => 0.03 }
45
- end
46
-
47
- it "should not be empty when multiple liness are added" do
48
- @completed_request.should_not be_empty
49
- end
50
-
51
- it "should be a completed request" do
52
- @completed_request.should be_completed
53
- end
54
-
55
- it "should recognize all line types" do
56
- [:first, :test, :last].each { |type| @completed_request.should =~ type }
57
- end
58
-
59
- it "should detect the correct field value" do
60
- @completed_request[:name].should == 'first line!'
61
- @completed_request[:time].should == 0.03
62
- end
63
-
64
- it "should detect the first matching field value" do
65
- @completed_request.first(:test_capture).should == 'testing'
66
- end
67
-
68
- it "should detect the every matching field value" do
69
- @completed_request.every(:test_capture).should == ['testing', "testing some more"]
70
- end
71
-
72
- end
@@ -1,9 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
- require 'request_log_analyzer/aggregator/summarizer'
3
-
4
- describe RequestLogAnalyzer::Aggregator::Summarizer do
5
-
6
- before(:each) do
7
- end
8
-
9
- end