request-log-analyzer 1.0.2 → 1.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +10 -0
- data/DESIGN.rdoc +41 -0
- data/LICENSE +4 -4
- data/README.rdoc +38 -0
- data/Rakefile +6 -3
- data/bin/request-log-analyzer +70 -72
- data/lib/cli/command_line_arguments.rb +53 -53
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +43 -0
- data/lib/cli/progressbar.rb +166 -189
- data/lib/cli/tools.rb +49 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
- data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
- data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
- data/lib/request_log_analyzer/controller.rb +251 -98
- data/lib/request_log_analyzer/database/base.rb +114 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/database.rb +102 -0
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
- data/lib/request_log_analyzer/file_format/apache.rb +147 -0
- data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
- data/lib/request_log_analyzer/file_format/merb.rb +65 -29
- data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
- data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
- data/lib/request_log_analyzer/file_format/rack.rb +9 -0
- data/lib/request_log_analyzer/file_format/rails.rb +164 -78
- data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
- data/lib/request_log_analyzer/file_format.rb +252 -58
- data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
- data/lib/request_log_analyzer/filter/field.rb +19 -13
- data/lib/request_log_analyzer/filter/timespan.rb +25 -12
- data/lib/request_log_analyzer/filter.rb +30 -0
- data/lib/request_log_analyzer/line_definition.rb +69 -96
- data/lib/request_log_analyzer/log_processor.rb +31 -53
- data/lib/request_log_analyzer/mailer.rb +65 -0
- data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
- data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
- data/lib/request_log_analyzer/output/html.rb +187 -0
- data/lib/request_log_analyzer/output.rb +117 -0
- data/lib/request_log_analyzer/request.rb +125 -40
- data/lib/request_log_analyzer/source/database_loader.rb +87 -0
- data/lib/request_log_analyzer/source/log_parser.rb +297 -0
- data/lib/request_log_analyzer/source.rb +72 -0
- data/lib/request_log_analyzer/tracker/duration.rb +28 -64
- data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
- data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
- data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
- data/lib/request_log_analyzer/tracker.rb +101 -0
- data/lib/request_log_analyzer.rb +43 -13
- data/request-log-analyzer.gemspec +41 -0
- data/spec/database.yml +23 -0
- data/spec/fixtures/apache_combined.log +5 -0
- data/spec/fixtures/apache_common.log +10 -0
- data/spec/fixtures/decompression.log +12 -0
- data/spec/fixtures/decompression.log.bz2 +0 -0
- data/spec/fixtures/decompression.log.gz +0 -0
- data/spec/fixtures/decompression.log.zip +0 -0
- data/spec/fixtures/decompression.tar.gz +0 -0
- data/spec/fixtures/decompression.tgz +0 -0
- data/spec/fixtures/header_and_footer.log +6 -0
- data/spec/fixtures/merb_prefixed.log +9 -0
- data/spec/fixtures/mysql_slow_query.log +110 -0
- data/spec/fixtures/postgresql.log +2980 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/fixtures/sinatra.log +99 -0
- data/spec/integration/command_line_usage_spec.rb +84 -0
- data/spec/integration/mailer_spec.rb +179 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +152 -0
- data/spec/lib/helpers.rb +72 -0
- data/spec/lib/macros.rb +18 -0
- data/spec/lib/matchers.rb +77 -0
- data/spec/lib/mocks.rb +77 -0
- data/spec/lib/testing_format.rb +46 -0
- data/spec/spec_helper.rb +16 -59
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/aggregator/summarizer_spec.rb +26 -0
- data/spec/unit/controller/controller_spec.rb +41 -0
- data/spec/unit/controller/log_processor_spec.rb +18 -0
- data/spec/unit/database/base_class_spec.rb +183 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +133 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
- data/spec/unit/file_format/apache_format_spec.rb +203 -0
- data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
- data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
- data/spec/unit/file_format/file_format_api_spec.rb +69 -0
- data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
- data/spec/unit/file_format/line_definition_spec.rb +75 -0
- data/spec/unit/file_format/merb_format_spec.rb +52 -0
- data/spec/unit/file_format/mysql_format_spec.rb +154 -0
- data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
- data/spec/unit/file_format/rack_format_spec.rb +50 -0
- data/spec/unit/file_format/rails3_format_spec.rb +120 -0
- data/spec/unit/file_format/rails_format_spec.rb +180 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
- data/spec/unit/filter/field_filter_spec.rb +66 -0
- data/spec/unit/filter/filter_spec.rb +17 -0
- data/spec/unit/filter/timespan_filter_spec.rb +58 -0
- data/spec/unit/mailer_spec.rb +42 -0
- data/spec/unit/request_spec.rb +111 -0
- data/spec/unit/source/log_parser_spec.rb +119 -0
- data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
- data/spec/unit/tracker/tracker_api_spec.rb +125 -0
- data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
- data/tasks/github-gem.rake +290 -139
- data/tasks/request_log_analyzer.rake +23 -7
- metadata +221 -94
- data/DESIGN +0 -14
- data/HACKING +0 -7
- data/README.textile +0 -36
- data/lib/cli/bashcolorizer.rb +0 -60
- data/lib/request_log_analyzer/aggregator/database.rb +0 -148
- data/lib/request_log_analyzer/filter/base.rb +0 -29
- data/lib/request_log_analyzer/log_parser.rb +0 -173
- data/lib/request_log_analyzer/source/base.rb +0 -42
- data/lib/request_log_analyzer/source/log_file.rb +0 -170
- data/lib/request_log_analyzer/tracker/base.rb +0 -54
- data/lib/request_log_analyzer/tracker/category.rb +0 -71
- data/spec/controller_spec.rb +0 -40
- data/spec/database_inserter_spec.rb +0 -101
- data/spec/file_format_spec.rb +0 -78
- data/spec/file_formats/spec_format.rb +0 -26
- data/spec/filter_spec.rb +0 -137
- data/spec/line_definition_spec.rb +0 -124
- data/spec/log_parser_spec.rb +0 -68
- data/spec/log_processor_spec.rb +0 -57
- data/spec/merb_format_spec.rb +0 -38
- data/spec/rails_format_spec.rb +0 -76
- data/spec/request_spec.rb +0 -72
- data/spec/summarizer_spec.rb +0 -9
- data/tasks/rspec.rake +0 -6
data/spec/file_format_spec.rb
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::FileFormat, :format_definition do
|
|
4
|
-
|
|
5
|
-
before(:each) do
|
|
6
|
-
@first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
|
|
7
|
-
@second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
it "should specify lines with a hash" do
|
|
11
|
-
|
|
12
|
-
@first_file_format.new.should have(0).line_definitions
|
|
13
|
-
|
|
14
|
-
@first_file_format.format_definition do |line|
|
|
15
|
-
line.hash_test :regexp => /test/, :captures => []
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
@format_instance = @first_file_format.new
|
|
19
|
-
@format_instance.should have(1).line_definitions
|
|
20
|
-
@format_instance.line_definitions[:hash_test].should_not be_nil
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
it "should specift lines directly" do
|
|
24
|
-
@first_file_format.new.should have(0).line_definitions
|
|
25
|
-
|
|
26
|
-
@first_file_format.format_definition.direct_test do |line|
|
|
27
|
-
line.regexp = /test/
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
@first_file_format.new.line_definitions[:direct_test].should_not be_nil
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
it "specify lines with a block" do
|
|
34
|
-
|
|
35
|
-
@first_file_format.new.should have(0).line_definitions
|
|
36
|
-
|
|
37
|
-
@first_file_format.format_definition do |format|
|
|
38
|
-
format.block_test do |line|
|
|
39
|
-
line.regexp = /test/
|
|
40
|
-
line.captures = []
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
@first_file_format.new.line_definitions[:block_test].should_not be_nil
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
it "should define lines only for itself" do
|
|
48
|
-
|
|
49
|
-
@first_file_format.format_definition do |line|
|
|
50
|
-
line.first_test :regexp => /test/, :captures => []
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
@second_file_format.format_definition do |line|
|
|
54
|
-
line.second_test :regexp => /test/, :captures => []
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
@first_file_format.new.should have(1).line_definitions
|
|
58
|
-
@first_file_format.new.line_definitions[:first_test].should_not be_nil
|
|
59
|
-
@second_file_format.new.should have(1).line_definitions
|
|
60
|
-
@second_file_format.new.line_definitions[:second_test].should_not be_nil
|
|
61
|
-
end
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
describe RequestLogAnalyzer::FileFormat, :load do
|
|
65
|
-
|
|
66
|
-
include RequestLogAnalyzerSpecHelper
|
|
67
|
-
|
|
68
|
-
it "should load a predefined file format from the /file_format dir" do
|
|
69
|
-
@file_format = RequestLogAnalyzer::FileFormat.load(:rails)
|
|
70
|
-
@file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
it "should load a provided format file" do
|
|
74
|
-
@file_format = RequestLogAnalyzer::FileFormat.load(format_file(:spec_format))
|
|
75
|
-
@file_format.should be_kind_of(SpecFormat)
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
end
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
class SpecFormat < RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
|
-
format_definition.first do |line|
|
|
4
|
-
line.header = true
|
|
5
|
-
line.teaser = /processing /
|
|
6
|
-
line.regexp = /processing request (\d+)/
|
|
7
|
-
line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
format_definition.test do |line|
|
|
11
|
-
line.teaser = /testing /
|
|
12
|
-
line.regexp = /testing is (\w+)/
|
|
13
|
-
line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
format_definition.last do |line|
|
|
17
|
-
line.footer = true
|
|
18
|
-
line.teaser = /finishing /
|
|
19
|
-
line.regexp = /finishing request (\d+)/
|
|
20
|
-
line.captures = [{ :name => :request_no, :type => :integer}]
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
report do |analyze|
|
|
24
|
-
analyze.category :test_capture, :title => 'What is testing exactly?'
|
|
25
|
-
end
|
|
26
|
-
end
|
data/spec/filter_spec.rb
DELETED
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
|
|
3
|
-
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
|
|
4
|
-
|
|
5
|
-
describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
|
|
6
|
-
include RequestLogAnalyzerSpecHelper
|
|
7
|
-
|
|
8
|
-
before(:each) do
|
|
9
|
-
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
|
|
10
|
-
@filter.prepare
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "should reject a request before the after date" do
|
|
14
|
-
@filter.filter(request(:timestamp => 20081212000000)).should be_nil
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should reject a request after the before date" do
|
|
18
|
-
@filter.filter(request(:timestamp => 20090303000000)).should be_nil
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "should accept a request between the after and before dates" do
|
|
22
|
-
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
|
|
27
|
-
include RequestLogAnalyzerSpecHelper
|
|
28
|
-
|
|
29
|
-
before(:each) do
|
|
30
|
-
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))
|
|
31
|
-
@filter.prepare
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
it "should accept a request before the after date" do
|
|
35
|
-
@filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it "should reject a request after the before date" do
|
|
39
|
-
@filter.filter(request(:timestamp => 20090303000000)).should be_nil
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
it "should accept a request between the after and before dates" do
|
|
43
|
-
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
|
|
48
|
-
include RequestLogAnalyzerSpecHelper
|
|
49
|
-
|
|
50
|
-
before(:each) do
|
|
51
|
-
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
|
|
52
|
-
@filter.prepare
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
it "should reject a request before the after date" do
|
|
56
|
-
@filter.filter(request(:timestamp => 20081212000000)).should be_nil
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
it "should accept a request after the before date" do
|
|
60
|
-
@filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
it "should accept a request between the after and before dates" do
|
|
64
|
-
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
|
|
69
|
-
include RequestLogAnalyzerSpecHelper
|
|
70
|
-
|
|
71
|
-
before(:each) do
|
|
72
|
-
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
|
|
73
|
-
@filter.prepare
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
it "should reject a request if the field value does not match" do
|
|
77
|
-
@filter.filter(request(:test => 'not test')).should be_nil
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
it "should reject a request if the field name does not match" do
|
|
81
|
-
@filter.filter(request(:testing => 'test')).should be_nil
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
it "should accept a request if the both name and value match" do
|
|
85
|
-
@filter.filter(request(:test => 'test')).should_not be_nil
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
it "should accept a request if the value is not the first value" do
|
|
89
|
-
@filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
|
|
94
|
-
include RequestLogAnalyzerSpecHelper
|
|
95
|
-
|
|
96
|
-
before(:each) do
|
|
97
|
-
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
|
|
98
|
-
@filter.prepare
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
it "should accept a request if the field value does not match" do
|
|
102
|
-
@filter.filter(request(:test => 'not test')).should_not be_nil
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
it "should accept a request if the field name does not match" do
|
|
106
|
-
@filter.filter(request(:testing => 'test')).should_not be_nil
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
it "should reject a request if the both name and value match" do
|
|
110
|
-
@filter.filter(request(:test => 'test')).should be_nil
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
it "should reject a request if the value is not the first value" do
|
|
114
|
-
@filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
|
|
119
|
-
include RequestLogAnalyzerSpecHelper
|
|
120
|
-
|
|
121
|
-
before(:each) do
|
|
122
|
-
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
|
|
123
|
-
@filter.prepare
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
it "should reject a request if the field value does not match" do
|
|
127
|
-
@filter.filter(request(:test => 'a working test')).should_not be_nil
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
it "should reject a request if the field name does not match" do
|
|
131
|
-
@filter.filter(request(:testing => 'test')).should be_nil
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
it "should accept a request if the value is not the first value" do
|
|
135
|
-
@filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
|
|
136
|
-
end
|
|
137
|
-
end
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::LineDefinition, :parsing do
|
|
4
|
-
|
|
5
|
-
before(:each) do
|
|
6
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
7
|
-
:teaser => /Testing /,
|
|
8
|
-
:regexp => /Testing (\w+), tries\: (\d+)/,
|
|
9
|
-
:captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
|
|
10
|
-
})
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "should return false on an unmatching line" do
|
|
14
|
-
(@line_definition =~ "nonmatching").should be_false
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should return false when only the teaser matches" do
|
|
18
|
-
(@line_definition =~ "Testing LineDefinition").should be_false
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "should return a hash if the line matches" do
|
|
22
|
-
(@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
it "should return a hash with all captures set" do
|
|
26
|
-
hash = @line_definition.matches("Testing LineDefinition, tries: 123")
|
|
27
|
-
hash[:what].should == "LineDefinition"
|
|
28
|
-
hash[:tries].should == 123
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
it "should return a hash with :line_type set" do
|
|
32
|
-
@line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
|
|
37
|
-
before(:each) do
|
|
38
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
39
|
-
:teaser => /Anonymize /,
|
|
40
|
-
:regexp => /Anonymize (\w+)!/,
|
|
41
|
-
:captures => [{ :name => :what, :type => :string }]
|
|
42
|
-
})
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
it "should return nil if the teaser does not match" do
|
|
46
|
-
@line_definition.anonymize("Nonsense").should be_nil
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
it "should return nil if no teaser exists and the regexp doesn't match" do
|
|
50
|
-
line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
51
|
-
:regexp => /Anonymize!/, :captures => []})
|
|
52
|
-
|
|
53
|
-
line_definition.anonymize('nonsense').should be_nil
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
it "should return itself if only the teaser matches" do
|
|
57
|
-
@line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
|
|
61
|
-
@line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
it "should return a string if the line matches" do
|
|
65
|
-
@line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
it "should not anonymize :what" do
|
|
69
|
-
@line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
|
|
74
|
-
|
|
75
|
-
it "should anonymize completely if anonymize is true" do
|
|
76
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
77
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
|
|
78
|
-
|
|
79
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
it "should anonymize a URL" do
|
|
83
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
84
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
|
|
85
|
-
|
|
86
|
-
@line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
it "should anonymize an IP address" do
|
|
90
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
91
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
|
|
92
|
-
|
|
93
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
it "should anonymize completely if the anonymizer is unknown" do
|
|
97
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
98
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
|
|
99
|
-
|
|
100
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
it "should anonymize an integer slightly" do
|
|
104
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
105
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
|
106
|
-
|
|
107
|
-
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
it "should anonymize an integer slightly" do
|
|
111
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
112
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
|
113
|
-
|
|
114
|
-
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
it "should anonymize an double slightly" do
|
|
118
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
119
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
|
|
120
|
-
|
|
121
|
-
@line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
end
|
data/spec/log_parser_spec.rb
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::LogParser, :requests do
|
|
4
|
-
include RequestLogAnalyzerSpecHelper
|
|
5
|
-
|
|
6
|
-
before(:each) do
|
|
7
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
it "should have multiple line definitions" do
|
|
11
|
-
@log_parser.file_format.line_definitions.length.should >= 2
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it "should have a valid language" do
|
|
15
|
-
@log_parser.file_format.should be_valid
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
it "should parse more lines than requests" do
|
|
19
|
-
@log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
|
|
20
|
-
@log_parser.parse_file(log_fixture(:test_language_combined))
|
|
21
|
-
@log_parser.parsed_lines.should > 2
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
it "should parse requests spanned over multiple files" do
|
|
25
|
-
@log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).once
|
|
26
|
-
@log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it "should parse all request values when spanned over multiple files" do
|
|
30
|
-
@log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
|
|
31
|
-
request.lines.should have(4).items
|
|
32
|
-
|
|
33
|
-
request[:request_no].should == 1
|
|
34
|
-
request[:test_capture].should == "amazing"
|
|
35
|
-
end
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
it "should parse a stream and find valid requests" do
|
|
39
|
-
io = File.new(log_fixture(:test_file_format), 'r')
|
|
40
|
-
@log_parser.parse_io(io) do |request|
|
|
41
|
-
request.should be_kind_of(RequestLogAnalyzer::Request)
|
|
42
|
-
request.should =~ :test
|
|
43
|
-
request[:test_capture].should_not be_nil
|
|
44
|
-
end
|
|
45
|
-
io.close
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
describe RequestLogAnalyzer::LogParser, :warnings do
|
|
51
|
-
include RequestLogAnalyzerSpecHelper
|
|
52
|
-
|
|
53
|
-
before(:each) do
|
|
54
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
it "should warn about teaser matching problems" do
|
|
58
|
-
@log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
|
|
59
|
-
@log_parser.parse_file(log_fixture(:test_file_format))
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
it "should warn about unmatching request headers and footers" do
|
|
63
|
-
@log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
|
|
64
|
-
@log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
|
|
65
|
-
@log_parser.should_not_receive(:handle_request)
|
|
66
|
-
@log_parser.parse_file(log_fixture(:test_order))
|
|
67
|
-
end
|
|
68
|
-
end
|
data/spec/log_processor_spec.rb
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
require 'request_log_analyzer/log_processor'
|
|
3
|
-
|
|
4
|
-
describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
|
|
5
|
-
|
|
6
|
-
include RequestLogAnalyzerSpecHelper
|
|
7
|
-
|
|
8
|
-
before(:each) do
|
|
9
|
-
@log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
|
|
10
|
-
@alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "should keep a junk line if :keep_junk_lines is true" do
|
|
14
|
-
@alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should remove a junk line" do
|
|
18
|
-
@log_anonymizer.anonymize_line("junk line\n").should be_empty
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "should keep a teaser line intact" do
|
|
22
|
-
@log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
it "should discard a teaser line if discard_teaser_line is true" do
|
|
26
|
-
@alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it "should keep a matching line intact if no anonymizing is declared" do
|
|
30
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
it "should anonymize values completely if requested" do
|
|
34
|
-
@alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
it "should anonymize values slightly if requested" do
|
|
38
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
|
|
43
|
-
|
|
44
|
-
include RequestLogAnalyzerSpecHelper
|
|
45
|
-
|
|
46
|
-
before(:each) do
|
|
47
|
-
@log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
it "should remove a junk line" do
|
|
51
|
-
@log_stripper.strip_line("junk line\n").should be_empty
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
it "should keep a teaser line intact" do
|
|
55
|
-
@log_stripper.strip_line("processing 1234\n").should be_empty
|
|
56
|
-
end
|
|
57
|
-
end
|
data/spec/merb_format_spec.rb
DELETED
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
# require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
#
|
|
3
|
-
# describe RequestLogAnalyzer::LogParser, "Merb" do
|
|
4
|
-
# include RequestLogAnalyzerSpecHelper
|
|
5
|
-
#
|
|
6
|
-
# before(:each) do
|
|
7
|
-
# @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
|
|
8
|
-
# end
|
|
9
|
-
#
|
|
10
|
-
# it "should have a valid language definitions" do
|
|
11
|
-
# @log_parser.file_format.should be_valid
|
|
12
|
-
# end
|
|
13
|
-
#
|
|
14
|
-
# it "should parse a stream and find valid requests" do
|
|
15
|
-
# File.open(log_fixture(:merb), 'r') do |io|
|
|
16
|
-
# @log_parser.parse_io(io) do |request|
|
|
17
|
-
# request.should be_kind_of(RequestLogAnalyzer::Request)
|
|
18
|
-
# end
|
|
19
|
-
# end
|
|
20
|
-
# end
|
|
21
|
-
#
|
|
22
|
-
# it "should find 11 completed requests" do
|
|
23
|
-
# @log_parser.should_receive(:handle_request).exactly(11).times
|
|
24
|
-
# @log_parser.parse_file(log_fixture(:merb))
|
|
25
|
-
# end
|
|
26
|
-
#
|
|
27
|
-
# it "should parse all details from a request correctly" do
|
|
28
|
-
# request = nil
|
|
29
|
-
# @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
|
|
30
|
-
#
|
|
31
|
-
# request.should be_completed
|
|
32
|
-
# request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008')
|
|
33
|
-
# request[:dispatch_time].should == 0.243424
|
|
34
|
-
# request[:after_filters_time].should == 6.9e-05
|
|
35
|
-
# request[:before_filters_time].should == 0.213213
|
|
36
|
-
# request[:action_time].should == 0.241652
|
|
37
|
-
# end
|
|
38
|
-
# end
|
data/spec/rails_format_spec.rb
DELETED
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::LogParser, "Rails" do
|
|
4
|
-
include RequestLogAnalyzerSpecHelper
|
|
5
|
-
|
|
6
|
-
before(:each) do
|
|
7
|
-
@log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
it "should have a valid language definitions" do
|
|
11
|
-
@log_parser.file_format.should be_valid
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it "should parse a stream and find valid requests" do
|
|
15
|
-
io = File.new(log_fixture(:rails_1x), 'r')
|
|
16
|
-
@log_parser.parse_io(io) do |request|
|
|
17
|
-
request.should be_kind_of(RequestLogAnalyzer::Request)
|
|
18
|
-
end
|
|
19
|
-
io.close
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it "should find 4 completed requests" do
|
|
23
|
-
@log_parser.should_not_receive(:warn)
|
|
24
|
-
@log_parser.should_receive(:handle_request).exactly(4).times
|
|
25
|
-
@log_parser.parse_file(log_fixture(:rails_1x))
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
it "should parse a Rails 2.2 request properly" do
|
|
29
|
-
@log_parser.should_not_receive(:warn)
|
|
30
|
-
@log_parser.parse_file(log_fixture(:rails_22)) do |request|
|
|
31
|
-
request.should =~ :processing
|
|
32
|
-
request.should =~ :completed
|
|
33
|
-
|
|
34
|
-
request[:controller].should == 'PageController'
|
|
35
|
-
request[:action].should == 'demo'
|
|
36
|
-
request[:url].should == 'http://www.example.coml/demo'
|
|
37
|
-
request[:status].should == 200
|
|
38
|
-
request[:duration].should == 0.614
|
|
39
|
-
request[:db].should == 0.031
|
|
40
|
-
request[:view].should == 0.120
|
|
41
|
-
end
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
it "should parse a syslog file with prefix correctly" do
|
|
45
|
-
@log_parser.should_not_receive(:warn)
|
|
46
|
-
@log_parser.parse_file(log_fixture(:syslog_1x)) do |request|
|
|
47
|
-
|
|
48
|
-
request.should be_completed
|
|
49
|
-
|
|
50
|
-
request[:controller].should == 'EmployeeController'
|
|
51
|
-
request[:action].should == 'index'
|
|
52
|
-
request[:url].should == 'http://example.com/employee.xml'
|
|
53
|
-
request[:status].should == 200
|
|
54
|
-
request[:duration].should == 0.21665
|
|
55
|
-
request[:db].should == 0.0
|
|
56
|
-
request[:view].should == 0.00926
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
it "should parse cached requests" do
|
|
61
|
-
@log_parser.should_not_receive(:warn)
|
|
62
|
-
@log_parser.parse_file(log_fixture(:rails_22_cached)) do |request|
|
|
63
|
-
request.should be_completed
|
|
64
|
-
request =~ :cache_hit
|
|
65
|
-
end
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
it "should detect unordered requests in the logs" do
|
|
69
|
-
@log_parser.should_not_receive(:handle_request)
|
|
70
|
-
# the first Processing-line will not give a warning, but the next one will
|
|
71
|
-
@log_parser.should_receive(:warn).with(:unclosed_request, anything).once
|
|
72
|
-
# Both Completed ;ines will give a warning
|
|
73
|
-
@log_parser.should_receive(:warn).with(:no_current_request, anything).twice
|
|
74
|
-
@log_parser.parse_file(log_fixture(:rails_unordered))
|
|
75
|
-
end
|
|
76
|
-
end
|
data/spec/request_spec.rb
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::Request, :incomplete_request do
|
|
4
|
-
|
|
5
|
-
include RequestLogAnalyzerSpecHelper
|
|
6
|
-
|
|
7
|
-
before(:each) do
|
|
8
|
-
@incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
|
|
9
|
-
@incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
it "should be single if only one line has been added" do
|
|
13
|
-
@incomplete_request.should_not be_empty
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
it "should not be a completed request" do
|
|
17
|
-
@incomplete_request.should_not be_completed
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
it "should take the line type of the first line as global line_type" do
|
|
21
|
-
@incomplete_request.lines[0][:line_type].should == :test
|
|
22
|
-
@incomplete_request.should =~ :test
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
it "should return the first field value" do
|
|
26
|
-
@incomplete_request[:test_capture].should == 'awesome!'
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it "should return nil if no such field is present" do
|
|
30
|
-
@incomplete_request[:nonexisting].should be_nil
|
|
31
|
-
end
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
describe RequestLogAnalyzer::Request, :completed_request do
|
|
36
|
-
|
|
37
|
-
include RequestLogAnalyzerSpecHelper
|
|
38
|
-
|
|
39
|
-
before(:each) do
|
|
40
|
-
@completed_request = RequestLogAnalyzer::Request.new(spec_format)
|
|
41
|
-
@completed_request << { :line_type => :first, :lineno => 1, :name => 'first line!' }
|
|
42
|
-
@completed_request << { :line_type => :test, :lineno => 4, :test_capture => 'testing' }
|
|
43
|
-
@completed_request << { :line_type => :test, :lineno => 7, :test_capture => 'testing some more' }
|
|
44
|
-
@completed_request << { :line_type => :last, :lineno => 10, :time => 0.03 }
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
it "should not be empty when multiple liness are added" do
|
|
48
|
-
@completed_request.should_not be_empty
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
it "should be a completed request" do
|
|
52
|
-
@completed_request.should be_completed
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
it "should recognize all line types" do
|
|
56
|
-
[:first, :test, :last].each { |type| @completed_request.should =~ type }
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
it "should detect the correct field value" do
|
|
60
|
-
@completed_request[:name].should == 'first line!'
|
|
61
|
-
@completed_request[:time].should == 0.03
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
it "should detect the first matching field value" do
|
|
65
|
-
@completed_request.first(:test_capture).should == 'testing'
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
it "should detect the every matching field value" do
|
|
69
|
-
@completed_request.every(:test_capture).should == ['testing', "testing some more"]
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
end
|