request-log-analyzer 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/DESIGN +24 -10
  2. data/README.rdoc +1 -1
  3. data/Rakefile +1 -2
  4. data/bin/request-log-analyzer +4 -38
  5. data/lib/cli/progressbar.rb +2 -19
  6. data/lib/cli/tools.rb +46 -0
  7. data/lib/request_log_analyzer/aggregator/database.rb +33 -8
  8. data/lib/request_log_analyzer/aggregator/echo.rb +1 -0
  9. data/lib/request_log_analyzer/aggregator/summarizer.rb +22 -16
  10. data/lib/request_log_analyzer/controller.rb +21 -5
  11. data/lib/request_log_analyzer/file_format/merb.rb +22 -9
  12. data/lib/request_log_analyzer/file_format/rails.rb +37 -23
  13. data/lib/request_log_analyzer/file_format/rails_development.rb +33 -88
  14. data/lib/request_log_analyzer/file_format.rb +50 -18
  15. data/lib/request_log_analyzer/filter/{anonimize.rb → anonymize.rb} +3 -8
  16. data/lib/request_log_analyzer/filter/field.rb +7 -6
  17. data/lib/request_log_analyzer/filter/timespan.rb +7 -3
  18. data/lib/request_log_analyzer/filter.rb +0 -4
  19. data/lib/request_log_analyzer/line_definition.rb +22 -88
  20. data/lib/request_log_analyzer/log_processor.rb +5 -25
  21. data/lib/request_log_analyzer/output/fixed_width.rb +14 -3
  22. data/lib/request_log_analyzer/output/html.rb +1 -0
  23. data/lib/request_log_analyzer/request.rb +62 -4
  24. data/lib/request_log_analyzer/source/database.rb +76 -0
  25. data/lib/request_log_analyzer/source/log_parser.rb +104 -52
  26. data/lib/request_log_analyzer/source.rb +28 -6
  27. data/lib/request_log_analyzer/tracker/duration.rb +90 -15
  28. data/lib/request_log_analyzer/tracker/{category.rb → frequency.rb} +24 -12
  29. data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -9
  30. data/lib/request_log_analyzer/tracker/timespan.rb +15 -8
  31. data/lib/request_log_analyzer.rb +29 -16
  32. data/spec/integration/command_line_usage_spec.rb +71 -0
  33. data/spec/lib/helper.rb +33 -0
  34. data/spec/lib/mocks.rb +47 -0
  35. data/spec/lib/testing_format.rb +39 -0
  36. data/spec/spec_helper.rb +5 -25
  37. data/spec/{database_inserter_spec.rb → unit/aggregator/database_inserter_spec.rb} +40 -37
  38. data/spec/unit/aggregator/summarizer_spec.rb +28 -0
  39. data/spec/unit/controller/controller_spec.rb +43 -0
  40. data/spec/unit/controller/log_processor_spec.rb +20 -0
  41. data/spec/{file_format_spec.rb → unit/file_format/file_format_api_spec.rb} +18 -6
  42. data/spec/unit/file_format/line_definition_spec.rb +64 -0
  43. data/spec/{merb_format_spec.rb → unit/file_format/merb_format_spec.rb} +3 -3
  44. data/spec/{rails_format_spec.rb → unit/file_format/rails_format_spec.rb} +24 -15
  45. data/spec/unit/filter/anonymize_filter_spec.rb +22 -0
  46. data/spec/unit/filter/field_filter_spec.rb +69 -0
  47. data/spec/unit/filter/timespan_filter_spec.rb +61 -0
  48. data/spec/{log_parser_spec.rb → unit/source/log_parser_spec.rb} +9 -10
  49. data/spec/{request_spec.rb → unit/source/request_spec.rb} +5 -5
  50. data/spec/unit/tracker/duration_tracker_spec.rb +99 -0
  51. data/spec/unit/tracker/frequency_tracker_spec.rb +83 -0
  52. data/spec/unit/tracker/hourly_spread_spec.rb +75 -0
  53. data/spec/unit/tracker/timespan_tracker_spec.rb +65 -0
  54. data/spec/unit/tracker/tracker_api_test.rb +45 -0
  55. data/tasks/rspec.rake +12 -0
  56. metadata +56 -28
  57. data/spec/controller_spec.rb +0 -45
  58. data/spec/file_formats/spec_format.rb +0 -27
  59. data/spec/filter_spec.rb +0 -157
  60. data/spec/line_definition_spec.rb +0 -124
  61. data/spec/log_processor_spec.rb +0 -57
  62. data/spec/summarizer_spec.rb +0 -9
@@ -1,124 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
-
3
- describe RequestLogAnalyzer::LineDefinition, :parsing do
4
-
5
- before(:each) do
6
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
7
- :teaser => /Testing /,
8
- :regexp => /Testing (\w+), tries\: (\d+)/,
9
- :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
10
- })
11
- end
12
-
13
- it "should return false on an unmatching line" do
14
- (@line_definition =~ "nonmatching").should be_false
15
- end
16
-
17
- it "should return false when only the teaser matches" do
18
- (@line_definition =~ "Testing LineDefinition").should be_false
19
- end
20
-
21
- it "should return a hash if the line matches" do
22
- (@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
23
- end
24
-
25
- it "should return a hash with all captures set" do
26
- hash = @line_definition.matches("Testing LineDefinition, tries: 123")
27
- hash[:what].should == "LineDefinition"
28
- hash[:tries].should == 123
29
- end
30
-
31
- it "should return a hash with :line_type set" do
32
- @line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
33
- end
34
- end
35
-
36
- describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
37
- before(:each) do
38
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
39
- :teaser => /Anonymize /,
40
- :regexp => /Anonymize (\w+)!/,
41
- :captures => [{ :name => :what, :type => :string }]
42
- })
43
- end
44
-
45
- it "should return nil if the teaser does not match" do
46
- @line_definition.anonymize("Nonsense").should be_nil
47
- end
48
-
49
- it "should return nil if no teaser exists and the regexp doesn't match" do
50
- line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
51
- :regexp => /Anonymize!/, :captures => []})
52
-
53
- line_definition.anonymize('nonsense').should be_nil
54
- end
55
-
56
- it "should return itself if only the teaser matches" do
57
- @line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
58
- end
59
-
60
- it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
61
- @line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
62
- end
63
-
64
- it "should return a string if the line matches" do
65
- @line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
66
- end
67
-
68
- it "should not anonymize :what" do
69
- @line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
70
- end
71
- end
72
-
73
- describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
74
-
75
- it "should anonymize completely if anonymize is true" do
76
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
77
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
78
-
79
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
80
- end
81
-
82
- it "should anonymize a URL" do
83
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
84
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
85
-
86
- @line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
87
- end
88
-
89
- it "should anonymize an IP address" do
90
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
91
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
92
-
93
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
94
- end
95
-
96
- it "should anonymize completely if the anonymizer is unknown" do
97
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
98
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
99
-
100
- @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
101
- end
102
-
103
- it "should anonymize an integer slightly" do
104
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
105
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
106
-
107
- @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
108
- end
109
-
110
- it "should anonymize an integer slightly" do
111
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
112
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
113
-
114
- @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
115
- end
116
-
117
- it "should anonymize an double slightly" do
118
- @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
119
- :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
120
-
121
- @line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
122
- end
123
-
124
- end
@@ -1,57 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
- require 'request_log_analyzer/log_processor'
3
-
4
- describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
5
-
6
- include RequestLogAnalyzerSpecHelper
7
-
8
- before(:each) do
9
- @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
10
- @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
11
- end
12
-
13
- it "should keep a junk line if :keep_junk_lines is true" do
14
- @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
15
- end
16
-
17
- it "should remove a junk line" do
18
- @log_anonymizer.anonymize_line("junk line\n").should be_empty
19
- end
20
-
21
- it "should keep a teaser line intact" do
22
- @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
23
- end
24
-
25
- it "should discard a teaser line if discard_teaser_line is true" do
26
- @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
27
- end
28
-
29
- it "should keep a matching line intact if no anonymizing is declared" do
30
- @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
31
- end
32
-
33
- it "should anonymize values completely if requested" do
34
- @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
35
- end
36
-
37
- it "should anonymize values slightly if requested" do
38
- @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
39
- end
40
- end
41
-
42
- describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
43
-
44
- include RequestLogAnalyzerSpecHelper
45
-
46
- before(:each) do
47
- @log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
48
- end
49
-
50
- it "should remove a junk line" do
51
- @log_stripper.strip_line("junk line\n").should be_empty
52
- end
53
-
54
- it "should keep a teaser line intact" do
55
- @log_stripper.strip_line("processing 1234\n").should be_empty
56
- end
57
- end
@@ -1,9 +0,0 @@
1
- require File.dirname(__FILE__) + '/spec_helper'
2
- require 'request_log_analyzer/aggregator/summarizer'
3
-
4
- describe RequestLogAnalyzer::Aggregator::Summarizer do
5
-
6
- before(:each) do
7
- end
8
-
9
- end