request-log-analyzer 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/request-log-analyzer +0 -11
- data/lib/request_log_analyzer/aggregator/summarizer.rb +7 -3
- data/lib/request_log_analyzer/controller.rb +15 -3
- data/lib/request_log_analyzer/file_format.rb +44 -16
- data/lib/request_log_analyzer/file_format/merb.rb +5 -5
- data/lib/request_log_analyzer/file_format/rails.rb +35 -21
- data/lib/request_log_analyzer/file_format/rails_development.rb +27 -20
- data/lib/request_log_analyzer/filter/{anonimize.rb → anonymize.rb} +3 -5
- data/lib/request_log_analyzer/filter/field.rb +1 -5
- data/lib/request_log_analyzer/filter/timespan.rb +0 -2
- data/lib/request_log_analyzer/line_definition.rb +12 -88
- data/lib/request_log_analyzer/log_processor.rb +5 -25
- data/lib/request_log_analyzer/request.rb +56 -4
- data/lib/request_log_analyzer/source/log_parser.rb +3 -4
- data/lib/request_log_analyzer/tracker/{category.rb → frequency.rb} +2 -2
- data/spec/controller_spec.rb +38 -19
- data/spec/file_format_spec.rb +2 -2
- data/spec/file_formats/spec_format.rb +12 -5
- data/spec/filter_spec.rb +3 -3
- data/spec/line_definition_spec.rb +18 -85
- data/spec/log_parser_spec.rb +2 -3
- data/spec/log_processor_spec.rb +0 -38
- data/spec/merb_format_spec.rb +1 -1
- data/spec/rails_format_spec.rb +6 -5
- data/spec/spec_helper.rb +16 -0
- metadata +4 -4
@@ -4,24 +4,31 @@ class SpecFormat < RequestLogAnalyzer::FileFormat::Base
|
|
4
4
|
line.header = true
|
5
5
|
line.teaser = /processing /
|
6
6
|
line.regexp = /processing request (\d+)/
|
7
|
-
line.captures = [{ :name => :request_no, :type => :integer
|
7
|
+
line.captures = [{ :name => :request_no, :type => :integer }]
|
8
8
|
end
|
9
9
|
|
10
10
|
format_definition.test do |line|
|
11
11
|
line.teaser = /testing /
|
12
|
-
line.regexp = /testing is (\w+)
|
13
|
-
line.captures = [{ :name => :test_capture, :type => :
|
12
|
+
line.regexp = /testing is (\w+)(?: in (\d+\.\d+)ms)?/
|
13
|
+
line.captures = [{ :name => :test_capture, :type => :test_type },
|
14
|
+
{ :name => :duration, :type => :duration, :unit => :msec }]
|
14
15
|
end
|
15
16
|
|
16
17
|
format_definition.last do |line|
|
17
18
|
line.footer = true
|
18
19
|
line.teaser = /finishing /
|
19
20
|
line.regexp = /finishing request (\d+)/
|
20
|
-
line.captures = [{ :name => :request_no, :type => :integer}]
|
21
|
+
line.captures = [{ :name => :request_no, :type => :integer }]
|
21
22
|
end
|
22
23
|
|
23
24
|
report do |analyze|
|
24
|
-
analyze.
|
25
|
+
analyze.frequency :test_capture, :title => 'What is testing exactly?'
|
26
|
+
end
|
27
|
+
|
28
|
+
class Request < RequestLogAnalyzer::Request
|
29
|
+
def convert_test_type(value, definition)
|
30
|
+
"Testing is #{value}"
|
31
|
+
end
|
25
32
|
end
|
26
33
|
|
27
34
|
end
|
data/spec/filter_spec.rb
CHANGED
@@ -134,11 +134,11 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
|
|
134
134
|
end
|
135
135
|
end
|
136
136
|
|
137
|
-
describe RequestLogAnalyzer::Filter::
|
137
|
+
describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
|
138
138
|
include RequestLogAnalyzerSpecHelper
|
139
139
|
|
140
140
|
before(:each) do
|
141
|
-
@filter = RequestLogAnalyzer::Filter::
|
141
|
+
@filter = RequestLogAnalyzer::Filter::Anonymize.new(spec_format)
|
142
142
|
@filter.prepare
|
143
143
|
end
|
144
144
|
|
@@ -150,7 +150,7 @@ describe RequestLogAnalyzer::Filter::Anonimize, 'anonimize request' do
|
|
150
150
|
@filter.filter(request(:url => 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees')
|
151
151
|
end
|
152
152
|
|
153
|
-
it "should
|
153
|
+
it "should fuzz durations" do
|
154
154
|
@filter.filter(request(:duration => 100))[:duration].should_not eql(100)
|
155
155
|
end
|
156
156
|
|
@@ -22,103 +22,36 @@ describe RequestLogAnalyzer::LineDefinition, :parsing do
|
|
22
22
|
(@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
|
23
23
|
end
|
24
24
|
|
25
|
-
it "should return a hash with
|
25
|
+
it "should return a hash with :captures set to an array" do
|
26
26
|
hash = @line_definition.matches("Testing LineDefinition, tries: 123")
|
27
|
-
hash[:
|
28
|
-
hash[:
|
27
|
+
hash[:captures][0].should == "LineDefinition"
|
28
|
+
hash[:captures][1].should == "123"
|
29
29
|
end
|
30
30
|
|
31
|
-
it "should return a hash with :
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
|
37
|
-
before(:each) do
|
38
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
39
|
-
:teaser => /Anonymize /,
|
40
|
-
:regexp => /Anonymize (\w+)!/,
|
41
|
-
:captures => [{ :name => :what, :type => :string }]
|
42
|
-
})
|
43
|
-
end
|
44
|
-
|
45
|
-
it "should return nil if the teaser does not match" do
|
46
|
-
@line_definition.anonymize("Nonsense").should be_nil
|
47
|
-
end
|
48
|
-
|
49
|
-
it "should return nil if no teaser exists and the regexp doesn't match" do
|
50
|
-
line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
51
|
-
:regexp => /Anonymize!/, :captures => []})
|
52
|
-
|
53
|
-
line_definition.anonymize('nonsense').should be_nil
|
54
|
-
end
|
55
|
-
|
56
|
-
it "should return itself if only the teaser matches" do
|
57
|
-
@line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
|
58
|
-
end
|
59
|
-
|
60
|
-
it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
|
61
|
-
@line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
|
31
|
+
it "should return a hash with :line_definition set" do
|
32
|
+
@line_definition.matches("Testing LineDefinition, tries: 123")[:line_definition].should == @line_definition
|
62
33
|
end
|
63
|
-
|
64
|
-
it "should return a string if the line matches" do
|
65
|
-
@line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
|
66
|
-
end
|
67
|
-
|
68
|
-
it "should not anonymize :what" do
|
69
|
-
@line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
|
70
|
-
end
|
71
34
|
end
|
72
35
|
|
73
|
-
describe RequestLogAnalyzer::LineDefinition, :
|
74
|
-
|
75
|
-
it "should anonymize completely if anonymize is true" do
|
76
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
77
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
|
36
|
+
describe RequestLogAnalyzer::LineDefinition, :converting do
|
78
37
|
|
79
|
-
|
80
|
-
end
|
81
|
-
|
82
|
-
it "should anonymize a URL" do
|
83
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
84
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
|
85
|
-
|
86
|
-
@line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
|
87
|
-
end
|
88
|
-
|
89
|
-
it "should anonymize an IP address" do
|
90
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
91
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
|
92
|
-
|
93
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
|
94
|
-
end
|
38
|
+
include RequestLogAnalyzerSpecHelper
|
95
39
|
|
96
|
-
|
97
|
-
@
|
98
|
-
|
99
|
-
|
100
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
40
|
+
before(:each) do
|
41
|
+
@file_format = spec_format
|
42
|
+
@request = @file_format.create_request
|
101
43
|
end
|
102
44
|
|
103
|
-
it "should
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
45
|
+
it "should convert captures to a hash of converted values" do
|
46
|
+
hash = @file_format.line_definitions[:first].convert_captured_values(["456"], @request)
|
47
|
+
hash[:request_no].should == 456
|
108
48
|
end
|
109
|
-
|
110
|
-
it "should anonymize an integer slightly" do
|
111
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
112
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
113
49
|
|
114
|
-
|
50
|
+
it "should convert captures to a hash" do
|
51
|
+
hash = @file_format.line_definitions[:test].convert_captured_values(["willem", nil], @request)
|
52
|
+
hash[:test_capture].should == 'Testing is willem'
|
53
|
+
hash[:duration].should be_nil
|
115
54
|
end
|
116
|
-
|
117
|
-
it "should anonymize an double slightly" do
|
118
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
119
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
|
120
55
|
|
121
|
-
|
122
|
-
end
|
123
|
-
|
56
|
+
|
124
57
|
end
|
data/spec/log_parser_spec.rb
CHANGED
@@ -28,10 +28,9 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
|
|
28
28
|
|
29
29
|
it "should parse all request values when spanned over multiple files" do
|
30
30
|
@log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
|
31
|
-
request.lines.should have(4).items
|
32
|
-
|
31
|
+
request.lines.should have(4).items
|
33
32
|
request[:request_no].should == 1
|
34
|
-
request[:test_capture].should == "amazing"
|
33
|
+
request[:test_capture].should == "Testing is amazing" # Note the custom converter
|
35
34
|
end
|
36
35
|
end
|
37
36
|
|
data/spec/log_processor_spec.rb
CHANGED
@@ -1,44 +1,6 @@
|
|
1
1
|
require File.dirname(__FILE__) + '/spec_helper'
|
2
2
|
require 'request_log_analyzer/log_processor'
|
3
3
|
|
4
|
-
describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
|
5
|
-
|
6
|
-
include RequestLogAnalyzerSpecHelper
|
7
|
-
|
8
|
-
before(:each) do
|
9
|
-
@log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
|
10
|
-
@alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should keep a junk line if :keep_junk_lines is true" do
|
14
|
-
@alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
|
15
|
-
end
|
16
|
-
|
17
|
-
it "should remove a junk line" do
|
18
|
-
@log_anonymizer.anonymize_line("junk line\n").should be_empty
|
19
|
-
end
|
20
|
-
|
21
|
-
it "should keep a teaser line intact" do
|
22
|
-
@log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
|
23
|
-
end
|
24
|
-
|
25
|
-
it "should discard a teaser line if discard_teaser_line is true" do
|
26
|
-
@alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
|
27
|
-
end
|
28
|
-
|
29
|
-
it "should keep a matching line intact if no anonymizing is declared" do
|
30
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
|
31
|
-
end
|
32
|
-
|
33
|
-
it "should anonymize values completely if requested" do
|
34
|
-
@alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
|
35
|
-
end
|
36
|
-
|
37
|
-
it "should anonymize values slightly if requested" do
|
38
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
4
|
describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
|
43
5
|
|
44
6
|
include RequestLogAnalyzerSpecHelper
|
data/spec/merb_format_spec.rb
CHANGED
@@ -29,7 +29,7 @@ describe RequestLogAnalyzer::Source::LogParser, :merb do
|
|
29
29
|
@log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
|
30
30
|
|
31
31
|
request.should be_completed
|
32
|
-
|
32
|
+
request[:timestamp].should == 20080829111023 # 'Fri Aug 29 11:10:23 +0200 2008'
|
33
33
|
request[:dispatch_time].should == 0.243424
|
34
34
|
request[:after_filters_time].should == 6.9e-05
|
35
35
|
request[:before_filters_time].should == 0.213213
|
data/spec/rails_format_spec.rb
CHANGED
@@ -80,6 +80,7 @@ describe "RequestLogAnalyzer::FileFormat::RailsDevelopment - Rails with developm
|
|
80
80
|
|
81
81
|
before(:each) do
|
82
82
|
@file_format = RequestLogAnalyzer::FileFormat.load(:rails_development)
|
83
|
+
@request = @file_format.create_request
|
83
84
|
end
|
84
85
|
|
85
86
|
it "should have a valid language definitions" do
|
@@ -87,33 +88,33 @@ describe "RequestLogAnalyzer::FileFormat::RailsDevelopment - Rails with developm
|
|
87
88
|
end
|
88
89
|
|
89
90
|
it "should parse a rendered line" do
|
90
|
-
info = @file_format.line_definitions[:rendered].
|
91
|
+
info = @file_format.line_definitions[:rendered].match_for("Rendered layouts/_footer (2.9ms)", @request)
|
91
92
|
info[:render_file].should == 'layouts/_footer'
|
92
93
|
info[:render_duration].should == 0.0029
|
93
94
|
end
|
94
95
|
|
95
96
|
it "should parse a query executed line with colors" do
|
96
|
-
info = @file_format.line_definitions[:query_executed].
|
97
|
+
info = @file_format.line_definitions[:query_executed].match_for(" [4;36;1mUser Load (0.4ms)[0m [0;1mSELECT * FROM `users` WHERE (`users`.`id` = 18205844) [0m", @request)
|
97
98
|
info[:query_class].should == 'User'
|
98
99
|
info[:query_duration].should == 0.0004
|
99
100
|
info[:query_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 18205844)'
|
100
101
|
end
|
101
102
|
|
102
103
|
it "should parse a query executed line without colors" do
|
103
|
-
info = @file_format.line_definitions[:query_executed].
|
104
|
+
info = @file_format.line_definitions[:query_executed].match_for(" User Load (0.4ms) SELECT * FROM `users` WHERE (`users`.`id` = 18205844) ", @request)
|
104
105
|
info[:query_class].should == 'User'
|
105
106
|
info[:query_duration].should == 0.0004
|
106
107
|
info[:query_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 18205844)'
|
107
108
|
end
|
108
109
|
|
109
110
|
it "should parse a cached query line with colors" do
|
110
|
-
info = @file_format.line_definitions[:query_cached].
|
111
|
+
info = @file_format.line_definitions[:query_cached].match_for(' [4;35;1mCACHE (0.0ms)[0m [0mSELECT * FROM `users` WHERE (`users`.`id` = 0) [0m', @request)
|
111
112
|
info[:cached_duration].should == 0.0
|
112
113
|
info[:cached_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 0)'
|
113
114
|
end
|
114
115
|
|
115
116
|
it "should parse a cached query line without colors" do
|
116
|
-
info = @file_format.line_definitions[:query_cached].
|
117
|
+
info = @file_format.line_definitions[:query_cached].match_for(' CACHE (0.0ms) SELECT * FROM `users` WHERE (`users`.`id` = 0) ', @request)
|
117
118
|
info[:cached_duration].should == 0.0
|
118
119
|
info[:cached_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 0)'
|
119
120
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -28,6 +28,22 @@ module RequestLogAnalyzerSpecHelper
|
|
28
28
|
format.create_request(fields)
|
29
29
|
end
|
30
30
|
end
|
31
|
+
|
32
|
+
def mock_io
|
33
|
+
mio = mock('IO')
|
34
|
+
mio.stub!(:print)
|
35
|
+
mio.stub!(:puts)
|
36
|
+
mio.stub!(:write)
|
37
|
+
return mio
|
38
|
+
end
|
39
|
+
|
40
|
+
def mock_output
|
41
|
+
output = mock('RequestLogAnalyzer::Output')
|
42
|
+
output.stub!(:header)
|
43
|
+
output.stub!(:footer)
|
44
|
+
output.stub!(:io).and_return(mock_io)
|
45
|
+
return output
|
46
|
+
end
|
31
47
|
|
32
48
|
end
|
33
49
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: request-log-analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Willem van Bergen
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2009-01-
|
13
|
+
date: 2009-01-24 00:00:00 +01:00
|
14
14
|
default_executable: request-log-analyzer
|
15
15
|
dependencies: []
|
16
16
|
|
@@ -49,7 +49,7 @@ files:
|
|
49
49
|
- lib/request_log_analyzer/file_format/rails_development.rb
|
50
50
|
- lib/request_log_analyzer/filter
|
51
51
|
- lib/request_log_analyzer/filter.rb
|
52
|
-
- lib/request_log_analyzer/filter/
|
52
|
+
- lib/request_log_analyzer/filter/anonymize.rb
|
53
53
|
- lib/request_log_analyzer/filter/field.rb
|
54
54
|
- lib/request_log_analyzer/filter/timespan.rb
|
55
55
|
- lib/request_log_analyzer/line_definition.rb
|
@@ -64,8 +64,8 @@ files:
|
|
64
64
|
- lib/request_log_analyzer/source/log_parser.rb
|
65
65
|
- lib/request_log_analyzer/tracker
|
66
66
|
- lib/request_log_analyzer/tracker.rb
|
67
|
-
- lib/request_log_analyzer/tracker/category.rb
|
68
67
|
- lib/request_log_analyzer/tracker/duration.rb
|
68
|
+
- lib/request_log_analyzer/tracker/frequency.rb
|
69
69
|
- lib/request_log_analyzer/tracker/hourly_spread.rb
|
70
70
|
- lib/request_log_analyzer/tracker/timespan.rb
|
71
71
|
- spec
|