request-log-analyzer 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/DESIGN +24 -10
- data/README.rdoc +1 -1
- data/Rakefile +1 -2
- data/bin/request-log-analyzer +4 -38
- data/lib/cli/progressbar.rb +2 -19
- data/lib/cli/tools.rb +46 -0
- data/lib/request_log_analyzer/aggregator/database.rb +33 -8
- data/lib/request_log_analyzer/aggregator/echo.rb +1 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +22 -16
- data/lib/request_log_analyzer/controller.rb +21 -5
- data/lib/request_log_analyzer/file_format/merb.rb +22 -9
- data/lib/request_log_analyzer/file_format/rails.rb +37 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +33 -88
- data/lib/request_log_analyzer/file_format.rb +50 -18
- data/lib/request_log_analyzer/filter/{anonimize.rb → anonymize.rb} +3 -8
- data/lib/request_log_analyzer/filter/field.rb +7 -6
- data/lib/request_log_analyzer/filter/timespan.rb +7 -3
- data/lib/request_log_analyzer/filter.rb +0 -4
- data/lib/request_log_analyzer/line_definition.rb +22 -88
- data/lib/request_log_analyzer/log_processor.rb +5 -25
- data/lib/request_log_analyzer/output/fixed_width.rb +14 -3
- data/lib/request_log_analyzer/output/html.rb +1 -0
- data/lib/request_log_analyzer/request.rb +62 -4
- data/lib/request_log_analyzer/source/database.rb +76 -0
- data/lib/request_log_analyzer/source/log_parser.rb +104 -52
- data/lib/request_log_analyzer/source.rb +28 -6
- data/lib/request_log_analyzer/tracker/duration.rb +90 -15
- data/lib/request_log_analyzer/tracker/{category.rb → frequency.rb} +24 -12
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -9
- data/lib/request_log_analyzer/tracker/timespan.rb +15 -8
- data/lib/request_log_analyzer.rb +29 -16
- data/spec/integration/command_line_usage_spec.rb +71 -0
- data/spec/lib/helper.rb +33 -0
- data/spec/lib/mocks.rb +47 -0
- data/spec/lib/testing_format.rb +39 -0
- data/spec/spec_helper.rb +5 -25
- data/spec/{database_inserter_spec.rb → unit/aggregator/database_inserter_spec.rb} +40 -37
- data/spec/unit/aggregator/summarizer_spec.rb +28 -0
- data/spec/unit/controller/controller_spec.rb +43 -0
- data/spec/unit/controller/log_processor_spec.rb +20 -0
- data/spec/{file_format_spec.rb → unit/file_format/file_format_api_spec.rb} +18 -6
- data/spec/unit/file_format/line_definition_spec.rb +64 -0
- data/spec/{merb_format_spec.rb → unit/file_format/merb_format_spec.rb} +3 -3
- data/spec/{rails_format_spec.rb → unit/file_format/rails_format_spec.rb} +24 -15
- data/spec/unit/filter/anonymize_filter_spec.rb +22 -0
- data/spec/unit/filter/field_filter_spec.rb +69 -0
- data/spec/unit/filter/timespan_filter_spec.rb +61 -0
- data/spec/{log_parser_spec.rb → unit/source/log_parser_spec.rb} +9 -10
- data/spec/{request_spec.rb → unit/source/request_spec.rb} +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +99 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +83 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +75 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +65 -0
- data/spec/unit/tracker/tracker_api_test.rb +45 -0
- data/tasks/rspec.rake +12 -0
- metadata +56 -28
- data/spec/controller_spec.rb +0 -45
- data/spec/file_formats/spec_format.rb +0 -27
- data/spec/filter_spec.rb +0 -157
- data/spec/line_definition_spec.rb +0 -124
- data/spec/log_processor_spec.rb +0 -57
- data/spec/summarizer_spec.rb +0 -9
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
|
|
3
|
-
describe RequestLogAnalyzer::LineDefinition, :parsing do
|
|
4
|
-
|
|
5
|
-
before(:each) do
|
|
6
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
7
|
-
:teaser => /Testing /,
|
|
8
|
-
:regexp => /Testing (\w+), tries\: (\d+)/,
|
|
9
|
-
:captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
|
|
10
|
-
})
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "should return false on an unmatching line" do
|
|
14
|
-
(@line_definition =~ "nonmatching").should be_false
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should return false when only the teaser matches" do
|
|
18
|
-
(@line_definition =~ "Testing LineDefinition").should be_false
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "should return a hash if the line matches" do
|
|
22
|
-
(@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
it "should return a hash with all captures set" do
|
|
26
|
-
hash = @line_definition.matches("Testing LineDefinition, tries: 123")
|
|
27
|
-
hash[:what].should == "LineDefinition"
|
|
28
|
-
hash[:tries].should == 123
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
it "should return a hash with :line_type set" do
|
|
32
|
-
@line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
|
|
37
|
-
before(:each) do
|
|
38
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
39
|
-
:teaser => /Anonymize /,
|
|
40
|
-
:regexp => /Anonymize (\w+)!/,
|
|
41
|
-
:captures => [{ :name => :what, :type => :string }]
|
|
42
|
-
})
|
|
43
|
-
end
|
|
44
|
-
|
|
45
|
-
it "should return nil if the teaser does not match" do
|
|
46
|
-
@line_definition.anonymize("Nonsense").should be_nil
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
it "should return nil if no teaser exists and the regexp doesn't match" do
|
|
50
|
-
line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
51
|
-
:regexp => /Anonymize!/, :captures => []})
|
|
52
|
-
|
|
53
|
-
line_definition.anonymize('nonsense').should be_nil
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
it "should return itself if only the teaser matches" do
|
|
57
|
-
@line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
|
|
61
|
-
@line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
it "should return a string if the line matches" do
|
|
65
|
-
@line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
it "should not anonymize :what" do
|
|
69
|
-
@line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
|
|
74
|
-
|
|
75
|
-
it "should anonymize completely if anonymize is true" do
|
|
76
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
77
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
|
|
78
|
-
|
|
79
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
it "should anonymize a URL" do
|
|
83
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
84
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
|
|
85
|
-
|
|
86
|
-
@line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
it "should anonymize an IP address" do
|
|
90
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
91
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
|
|
92
|
-
|
|
93
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
it "should anonymize completely if the anonymizer is unknown" do
|
|
97
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
98
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
|
|
99
|
-
|
|
100
|
-
@line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
|
|
101
|
-
end
|
|
102
|
-
|
|
103
|
-
it "should anonymize an integer slightly" do
|
|
104
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
105
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
|
106
|
-
|
|
107
|
-
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
it "should anonymize an integer slightly" do
|
|
111
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
112
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
|
|
113
|
-
|
|
114
|
-
@line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
it "should anonymize an double slightly" do
|
|
118
|
-
@line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
|
|
119
|
-
:regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
|
|
120
|
-
|
|
121
|
-
@line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
end
|
data/spec/log_processor_spec.rb
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
require File.dirname(__FILE__) + '/spec_helper'
|
|
2
|
-
require 'request_log_analyzer/log_processor'
|
|
3
|
-
|
|
4
|
-
describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
|
|
5
|
-
|
|
6
|
-
include RequestLogAnalyzerSpecHelper
|
|
7
|
-
|
|
8
|
-
before(:each) do
|
|
9
|
-
@log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
|
|
10
|
-
@alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
it "should keep a junk line if :keep_junk_lines is true" do
|
|
14
|
-
@alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
it "should remove a junk line" do
|
|
18
|
-
@log_anonymizer.anonymize_line("junk line\n").should be_empty
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it "should keep a teaser line intact" do
|
|
22
|
-
@log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
it "should discard a teaser line if discard_teaser_line is true" do
|
|
26
|
-
@alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
it "should keep a matching line intact if no anonymizing is declared" do
|
|
30
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
it "should anonymize values completely if requested" do
|
|
34
|
-
@alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
it "should anonymize values slightly if requested" do
|
|
38
|
-
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
|
|
43
|
-
|
|
44
|
-
include RequestLogAnalyzerSpecHelper
|
|
45
|
-
|
|
46
|
-
before(:each) do
|
|
47
|
-
@log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
it "should remove a junk line" do
|
|
51
|
-
@log_stripper.strip_line("junk line\n").should be_empty
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
it "should keep a teaser line intact" do
|
|
55
|
-
@log_stripper.strip_line("processing 1234\n").should be_empty
|
|
56
|
-
end
|
|
57
|
-
end
|