request-log-analyzer 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. data/DESIGN +14 -0
  2. data/HACKING +7 -0
  3. data/LICENSE +20 -0
  4. data/README.textile +36 -0
  5. data/Rakefile +5 -0
  6. data/bin/request-log-analyzer +123 -0
  7. data/lib/cli/bashcolorizer.rb +60 -0
  8. data/lib/cli/command_line_arguments.rb +301 -0
  9. data/lib/cli/progressbar.rb +236 -0
  10. data/lib/request_log_analyzer.rb +14 -0
  11. data/lib/request_log_analyzer/aggregator/base.rb +45 -0
  12. data/lib/request_log_analyzer/aggregator/database.rb +148 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
  15. data/lib/request_log_analyzer/controller.rb +201 -0
  16. data/lib/request_log_analyzer/file_format.rb +81 -0
  17. data/lib/request_log_analyzer/file_format/merb.rb +33 -0
  18. data/lib/request_log_analyzer/file_format/rails.rb +90 -0
  19. data/lib/request_log_analyzer/filter/base.rb +29 -0
  20. data/lib/request_log_analyzer/filter/field.rb +36 -0
  21. data/lib/request_log_analyzer/filter/timespan.rb +32 -0
  22. data/lib/request_log_analyzer/line_definition.rb +159 -0
  23. data/lib/request_log_analyzer/log_parser.rb +173 -0
  24. data/lib/request_log_analyzer/log_processor.rb +121 -0
  25. data/lib/request_log_analyzer/request.rb +95 -0
  26. data/lib/request_log_analyzer/source/base.rb +42 -0
  27. data/lib/request_log_analyzer/source/log_file.rb +170 -0
  28. data/lib/request_log_analyzer/tracker/base.rb +54 -0
  29. data/lib/request_log_analyzer/tracker/category.rb +71 -0
  30. data/lib/request_log_analyzer/tracker/duration.rb +81 -0
  31. data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
  32. data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
  33. data/spec/controller_spec.rb +40 -0
  34. data/spec/database_inserter_spec.rb +101 -0
  35. data/spec/file_format_spec.rb +78 -0
  36. data/spec/file_formats/spec_format.rb +26 -0
  37. data/spec/filter_spec.rb +137 -0
  38. data/spec/fixtures/merb.log +84 -0
  39. data/spec/fixtures/multiple_files_1.log +5 -0
  40. data/spec/fixtures/multiple_files_2.log +2 -0
  41. data/spec/fixtures/rails_1x.log +59 -0
  42. data/spec/fixtures/rails_22.log +12 -0
  43. data/spec/fixtures/rails_22_cached.log +10 -0
  44. data/spec/fixtures/rails_unordered.log +24 -0
  45. data/spec/fixtures/syslog_1x.log +5 -0
  46. data/spec/fixtures/test_file_format.log +13 -0
  47. data/spec/fixtures/test_language_combined.log +14 -0
  48. data/spec/fixtures/test_order.log +16 -0
  49. data/spec/line_definition_spec.rb +124 -0
  50. data/spec/log_parser_spec.rb +68 -0
  51. data/spec/log_processor_spec.rb +57 -0
  52. data/spec/merb_format_spec.rb +38 -0
  53. data/spec/rails_format_spec.rb +76 -0
  54. data/spec/request_spec.rb +72 -0
  55. data/spec/spec_helper.rb +67 -0
  56. data/spec/summarizer_spec.rb +9 -0
  57. data/tasks/github-gem.rake +177 -0
  58. data/tasks/request_log_analyzer.rake +10 -0
  59. data/tasks/rspec.rake +6 -0
  60. metadata +135 -0
@@ -0,0 +1,84 @@
1
+ ~ Loaded DEVELOPMENT Environment...
2
+ ~ Connecting to database...
3
+ ~ loading gem 'merb_datamapper' ...
4
+ ~ loading gem 'gettext' ...
5
+ ~ loading gem 'merb_helpers' ...
6
+ ~ loading gem 'merb-assets' ...
7
+ ~ loading gem 'merb-action-args' ...
8
+ ~ loading gem 'merb-mailer' ...
9
+ ~ loading gem 'merb_param_protection' ...
10
+ ~ loading gem 'merb_has_flash' ...
11
+ ~ loading gem 'merb_forgery_protection' ...
12
+ ~ loading gem 'dm-validations' ...
13
+ ~ loading gem 'dm-timestamps' ...
14
+ ~ loading gem 'dm-migrations' ...
15
+ ~ loading gem 'dm-aggregates' ...
16
+ ~ loading gem 'dm-adjust' ...
17
+ ~ loading gem 'dm-serializer' ...
18
+ ~ loading gem 'dm-constraints' ...
19
+ ~ loading gem 'dm-timeline' ...
20
+ ~ loading gem 'dm-searchable' ...
21
+ ~ loading gem 'dm-audited' ...
22
+ ~ loading gem 'lib/extensions' ...
23
+ ~ loading gem 'lib/authenticated_system/authenticated_dependencies' ...
24
+ ~ Compiling routes...
25
+ ~ Using 'share-nothing' cookie sessions (4kb limit per client)
26
+ ~ Using Mongrel adapter
27
+ ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
28
+ ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy", "method"=>"delete", "controller"=>"session"}
29
+ ~ Cookie deleted: auth_token => nil
30
+ ~ Redirecting to: / (302)
31
+ ~ {:dispatch_time=>0.243424, :after_filters_time=>6.9e-05, :before_filters_time=>0.213213, :action_time=>0.241652}
32
+ ~
33
+
34
+ ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
35
+ ~ Params: {"action"=>"index", "controller"=>"dashboard"}
36
+ ~ Redirecting to: /login (302)
37
+ ~ {:dispatch_time=>0.002649, :after_filters_time=>7.4e-05, :action_time=>0.001951}
38
+ ~
39
+
40
+ ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
41
+ ~ Params: {"action"=>"create", "controller"=>"session"}
42
+ ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
43
+ ~
44
+
45
+ ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
46
+ ~ Params: {"authenticity_token"=>"[FILTERED]", "action"=>"create", "controller"=>"session", "login"=>"username", "password"=>"[FILTERED]", "remember_me"=>"0"}
47
+ ~ Redirecting to: / (302)
48
+ ~ {:dispatch_time=>0.006652, :after_filters_time=>0.000143, :before_filters_time=>0.000861, :action_time=>0.006171}
49
+ ~
50
+
51
+ ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
52
+ ~ Params: {"action"=>"index", "controller"=>"dashboard"}
53
+ ~ Redirecting to: /dashboard (302)
54
+ ~ {:dispatch_time=>0.008241, :after_filters_time=>0.000126, :before_filters_time=>0.002632, :action_time=>0.007711}
55
+ ~
56
+
57
+ ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
58
+ ~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
59
+ ~ {:dispatch_time=>0.009458, :after_filters_time=>0.000103, :before_filters_time=>0.00266, :action_time=>0.008742}
60
+ ~
61
+
62
+ ~ Started request handling: Fri Aug 29 11:10:29 +0200 2008
63
+ ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
64
+ ~ {:dispatch_time=>0.102725, :after_filters_time=>0.000115, :before_filters_time=>0.00411, :action_time=>0.101836}
65
+ ~
66
+
67
+ ~ Started request handling: Fri Aug 29 11:10:30 +0200 2008
68
+ ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"organisations"}
69
+ ~ {:dispatch_time=>0.042575, :after_filters_time=>8.9e-05, :before_filters_time=>0.004267, :action_time=>0.041762}
70
+ ~
71
+
72
+ ~ Started request handling: Fri Aug 29 11:10:31 +0200 2008
73
+ ~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
74
+ ~ {:dispatch_time=>0.010311, :after_filters_time=>8.0e-05, :before_filters_time=>0.003195, :action_time=>0.009567}
75
+ ~
76
+
77
+ ~ Started request handling: Fri Aug 29 11:10:33 +0200 2008
78
+ ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
79
+ ~ {:dispatch_time=>0.012913, :after_filters_time=>7.1e-05, :before_filters_time=>0.004422, :action_time=>0.012141}
80
+ ~
81
+
82
+ ~ Started request handling: Fri Aug 29 11:10:35 +0200 2008
83
+ ~ Params: {"action"=>"new", "namespace"=>"dashboard", "controller"=>"employees"}
84
+ ~ {:dispatch_time=>0.013051, :after_filters_time=>7.8e-05, :before_filters_time=>0.003576, :action_time=>0.011773}
@@ -0,0 +1,5 @@
1
+ initializing
2
+
3
+ processing request 1
4
+
5
+ testing is amazing
@@ -0,0 +1,2 @@
1
+ testing is cool
2
+ finishing request 1
@@ -0,0 +1,59 @@
1
+ Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET]
2
+ Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ
3
+ QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz
4
+ ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291
5
+ bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z
6
+ MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp
7
+ ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50
8
+ MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd
9
+ Parameters: {"action"=>"index", "controller"=>"dashboard"}
10
+ Set language to: nl_NL
11
+ Rendering template within layouts/priscilla
12
+ Rendering dashboard/index
13
+ Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/]
14
+
15
+
16
+ Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
17
+ Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
18
+ SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv
19
+ bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy
20
+ eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A
21
+ bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v
22
+ ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK
23
+ bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3
24
+ Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"}
25
+ Set language to: nl_NL
26
+ Redirected to https://www.example.com/people/2545
27
+ Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek]
28
+
29
+
30
+ Processing PeopleController#show (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]
31
+ Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p
32
+ dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh
33
+ c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0
34
+ EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry
35
+ IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn
36
+ ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu
37
+ bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u
38
+ X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a
39
+ Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"}
40
+ Set language to: nl_NL
41
+ Rendering template within layouts/priscilla
42
+ Rendering people/show
43
+ person: John Doe, study_year: 2008/2009
44
+ Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545]
45
+
46
+
47
+ Processing PeopleController#picture (for 1.1.1.1 at 2008-08-14 21:16:35) [GET]
48
+ Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo
49
+ SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv
50
+ MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj
51
+ dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo
52
+ YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz
53
+ bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6
54
+ DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167
55
+ Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"}
56
+ Set language to: nl_NL
57
+ Rendering template within layouts/priscilla
58
+ Rendering people/picture
59
+ Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture]
@@ -0,0 +1,12 @@
1
+ Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
2
+ Parameters: {"action"=>"demo", "controller"=>"page"}
3
+ Logging in from session data...
4
+ Logged in as test@example.com
5
+ Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain:
6
+ Rendering template within layouts/demo
7
+ Rendering page/demo
8
+ Rendered shared/_analytics (0.2ms)
9
+ Rendered layouts/_actions (0.6ms)
10
+ Rendered layouts/_menu (2.2ms)
11
+ Rendered layouts/_tabbar (0.5ms)
12
+ Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo]
@@ -0,0 +1,10 @@
1
+ Processing CachedController#cached (for 1.1.1.1 at 2008-12-24 07:36:53) [GET]
2
+ Parameters: {"action"=>"cached", "controller"=>"cached"}
3
+ Logging in from session data...
4
+ Logging in using cookie...
5
+ Using locale: zh-Hans, http-accept: ["zh-CN", "zh-HK", "zh-TW", "en-US"], session: , det browser: zh-Hans, det domain: , user pref locale:
6
+ Referer: http://www.example.com/referer
7
+ Cached fragment hit: views/zh-Hans-www-cached-cached-all-CN--- (0.0ms)
8
+ Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
9
+ Filter chain halted as [#<ActionController::Filters::AroundFilter:0x2a999ad120 @identifier=nil, @kind=:filter, @options={:only=>#<Set: {"cached"}>, :if=>:not_logged_in?, :unless=>nil}, @method=#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>>] did_not_yield.
10
+ Completed in 3ms (View: 0, DB: 0) | 200 OK [http://www.example.com/cached/cached/]
@@ -0,0 +1,24 @@
1
+ Processing AccountController#dashboard (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
2
+ Parameters: {"action"=>"dashboard", "controller"=>"account", "first_use"=>"true"}
3
+ Logging in from session data...
4
+
5
+
6
+ Processing ProjectsController#new (for 1.1.1.1 at 2008-12-24 07:36:49) [GET]
7
+ Parameters: {"action"=>"new", "controller"=>"projects"}
8
+ Rendering template within layouts/default
9
+ Rendering account/dashboard
10
+ Logging in from session data...
11
+ Logging in using cookie...
12
+ Using locale: en-US, http-accept: [], session: , det browser: , det domain: , user pref locale:
13
+ Rendered shared/_maintenance (0.6ms)
14
+ Rendering template within layouts/templates/general_default/index.html.erb
15
+ Rendered projects/_recent_designs (4.3ms)
16
+ Rendered projects/_project (13.6ms)
17
+ Rendered projects/_projects (18.7ms)
18
+ Rendered layouts/_menu (1.4ms)
19
+ Completed in 36ms (View: 30, DB: 3) | 200 OK [http://www.example.com/projects/new]
20
+ Rendered layouts/_actions (0.3ms)
21
+ Rendered layouts/_menu (1.6ms)
22
+ Rendered layouts/_tabbar (1.9ms)
23
+ Rendered layouts/_footer (3.2ms)
24
+ Completed in 50ms (View: 41, DB: 4) | 200 OK [http://www.example.com/dashboard?first_use=true]
@@ -0,0 +1,5 @@
1
+ Jul 13 06:25:58 10.1.1.32 app_p [1957]: Processing EmployeeController#index (for 10.1.1.33 at 2008-07-13 06:25:58) [GET]
2
+ Jul 13 06:25:58 10.1.1.32 app_p [1957]: Session ID: bd1810833653be11c38ad1e5675635bd
3
+ Jul 13 06:25:58 10.1.1.32 app_p [1957]: Parameters: {"format"=>"xml", "action"=>"index}
4
+ Jul 13 06:25:58 10.1.1.32 app_p [1957]: Rendering employees
5
+ Jul 13 06:25:58 10.1.1.32 app_p [1957]: Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://example.com/employee.xml]
@@ -0,0 +1,13 @@
1
+ processing request 1
2
+ testing
3
+ testing methods
4
+ testing is cool
5
+ testing fixtures
6
+ testing is amazing
7
+ testing can be cumbersome
8
+ testing
9
+ nonsense
10
+ testing is
11
+
12
+ more nonsense
13
+ finishing request 1
@@ -0,0 +1,14 @@
1
+ initializing
2
+
3
+ processing request 1
4
+
5
+ testing is amazing
6
+ testing is cool
7
+
8
+ finishing request 1
9
+
10
+ asdfad # garbage
11
+ dsaads # garbage
12
+
13
+ processing request 1
14
+ finishing request 1
@@ -0,0 +1,16 @@
1
+ initializing
2
+
3
+ processing request 1
4
+
5
+ testing is amazing
6
+ testing is cool
7
+
8
+ processing request 1
9
+ testing is cool
10
+ finishing request 1
11
+
12
+ asdfad # garbage
13
+ dsaads # garbage
14
+
15
+
16
+ finishing request 1
@@ -0,0 +1,124 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe RequestLogAnalyzer::LineDefinition, :parsing do
4
+
5
+ before(:each) do
6
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
7
+ :teaser => /Testing /,
8
+ :regexp => /Testing (\w+), tries\: (\d+)/,
9
+ :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
10
+ })
11
+ end
12
+
13
+ it "should return false on an unmatching line" do
14
+ (@line_definition =~ "nonmatching").should be_false
15
+ end
16
+
17
+ it "should return false when only the teaser matches" do
18
+ (@line_definition =~ "Testing LineDefinition").should be_false
19
+ end
20
+
21
+ it "should return a hash if the line matches" do
22
+ (@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
23
+ end
24
+
25
+ it "should return a hash with all captures set" do
26
+ hash = @line_definition.matches("Testing LineDefinition, tries: 123")
27
+ hash[:what].should == "LineDefinition"
28
+ hash[:tries].should == 123
29
+ end
30
+
31
+ it "should return a hash with :line_type set" do
32
+ @line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
33
+ end
34
+ end
35
+
36
+ describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
37
+ before(:each) do
38
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
39
+ :teaser => /Anonymize /,
40
+ :regexp => /Anonymize (\w+)!/,
41
+ :captures => [{ :name => :what, :type => :string }]
42
+ })
43
+ end
44
+
45
+ it "should return nil if the teaser does not match" do
46
+ @line_definition.anonymize("Nonsense").should be_nil
47
+ end
48
+
49
+ it "should return nil if no teaser exists and the regexp doesn't match" do
50
+ line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
51
+ :regexp => /Anonymize!/, :captures => []})
52
+
53
+ line_definition.anonymize('nonsense').should be_nil
54
+ end
55
+
56
+ it "should return itself if only the teaser matches" do
57
+ @line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
58
+ end
59
+
60
+ it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
61
+ @line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
62
+ end
63
+
64
+ it "should return a string if the line matches" do
65
+ @line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
66
+ end
67
+
68
+ it "should not anonymize :what" do
69
+ @line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
70
+ end
71
+ end
72
+
73
+ describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
74
+
75
+ it "should anonymize completely if anonymize is true" do
76
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
77
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
78
+
79
+ @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
80
+ end
81
+
82
+ it "should anonymize a URL" do
83
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
84
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
85
+
86
+ @line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
87
+ end
88
+
89
+ it "should anonymize an IP address" do
90
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
91
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
92
+
93
+ @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
94
+ end
95
+
96
+ it "should anonymize completely if the anonymizer is unknown" do
97
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
98
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
99
+
100
+ @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
101
+ end
102
+
103
+ it "should anonymize an integer slightly" do
104
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
105
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
106
+
107
+ @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
108
+ end
109
+
110
+ it "should anonymize an integer slightly" do
111
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
112
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
113
+
114
+ @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
115
+ end
116
+
117
+ it "should anonymize an double slightly" do
118
+ @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
119
+ :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
120
+
121
+ @line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
122
+ end
123
+
124
+ end
@@ -0,0 +1,68 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+
3
+ describe RequestLogAnalyzer::LogParser, :requests do
4
+ include RequestLogAnalyzerSpecHelper
5
+
6
+ before(:each) do
7
+ @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
8
+ end
9
+
10
+ it "should have multiple line definitions" do
11
+ @log_parser.file_format.line_definitions.length.should >= 2
12
+ end
13
+
14
+ it "should have a valid language" do
15
+ @log_parser.file_format.should be_valid
16
+ end
17
+
18
+ it "should parse more lines than requests" do
19
+ @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
20
+ @log_parser.parse_file(log_fixture(:test_language_combined))
21
+ @log_parser.parsed_lines.should > 2
22
+ end
23
+
24
+ it "should parse requests spanned over multiple files" do
25
+ @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).once
26
+ @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
27
+ end
28
+
29
+ it "should parse all request values when spanned over multiple files" do
30
+ @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
31
+ request.lines.should have(4).items
32
+
33
+ request[:request_no].should == 1
34
+ request[:test_capture].should == "amazing"
35
+ end
36
+ end
37
+
38
+ it "should parse a stream and find valid requests" do
39
+ io = File.new(log_fixture(:test_file_format), 'r')
40
+ @log_parser.parse_io(io) do |request|
41
+ request.should be_kind_of(RequestLogAnalyzer::Request)
42
+ request.should =~ :test
43
+ request[:test_capture].should_not be_nil
44
+ end
45
+ io.close
46
+ end
47
+
48
+ end
49
+
50
+ describe RequestLogAnalyzer::LogParser, :warnings do
51
+ include RequestLogAnalyzerSpecHelper
52
+
53
+ before(:each) do
54
+ @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
55
+ end
56
+
57
+ it "should warn about teaser matching problems" do
58
+ @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
59
+ @log_parser.parse_file(log_fixture(:test_file_format))
60
+ end
61
+
62
+ it "should warn about unmatching request headers and footers" do
63
+ @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times
64
+ @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times
65
+ @log_parser.should_not_receive(:handle_request)
66
+ @log_parser.parse_file(log_fixture(:test_order))
67
+ end
68
+ end