wvanbergen-request-log-analyzer 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/DESIGN +14 -0
- data/HACKING +7 -0
- data/README.textile +9 -98
- data/Rakefile +2 -2
- data/bin/request-log-analyzer +1 -1
- data/lib/cli/bashcolorizer.rb +60 -0
- data/lib/cli/command_line_arguments.rb +301 -0
- data/lib/cli/progressbar.rb +236 -0
- data/lib/request_log_analyzer/aggregator/base.rb +51 -0
- data/lib/request_log_analyzer/aggregator/database.rb +97 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
- data/lib/request_log_analyzer/controller.rb +206 -0
- data/lib/request_log_analyzer/file_format/merb.rb +33 -0
- data/lib/request_log_analyzer/file_format/rails.rb +119 -0
- data/lib/request_log_analyzer/file_format.rb +77 -0
- data/lib/request_log_analyzer/filter/base.rb +29 -0
- data/lib/request_log_analyzer/filter/field.rb +36 -0
- data/lib/request_log_analyzer/filter/timespan.rb +32 -0
- data/lib/request_log_analyzer/line_definition.rb +159 -0
- data/lib/request_log_analyzer/log_parser.rb +183 -0
- data/lib/request_log_analyzer/log_processor.rb +121 -0
- data/lib/request_log_analyzer/request.rb +115 -0
- data/lib/request_log_analyzer/source/base.rb +42 -0
- data/lib/request_log_analyzer/source/log_file.rb +180 -0
- data/lib/request_log_analyzer/tracker/base.rb +54 -0
- data/lib/request_log_analyzer/tracker/category.rb +71 -0
- data/lib/request_log_analyzer/tracker/duration.rb +81 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
- data/spec/file_format_spec.rb +78 -0
- data/spec/file_formats/spec_format.rb +26 -0
- data/spec/filter_spec.rb +137 -0
- data/spec/log_processor_spec.rb +57 -0
- data/tasks/rspec.rake +6 -0
- metadata +53 -55
- data/TODO +0 -58
- data/bin/request-log-database +0 -81
- data/lib/base/log_parser.rb +0 -78
- data/lib/base/record_inserter.rb +0 -139
- data/lib/command_line/arguments.rb +0 -129
- data/lib/command_line/flag.rb +0 -51
- data/lib/merb_analyzer/log_parser.rb +0 -26
- data/lib/rails_analyzer/log_parser.rb +0 -35
- data/lib/rails_analyzer/record_inserter.rb +0 -39
- data/tasks/test.rake +0 -8
- data/test/log_fragments/fragment_1.log +0 -59
- data/test/log_fragments/fragment_2.log +0 -5
- data/test/log_fragments/fragment_3.log +0 -12
- data/test/log_fragments/fragment_4.log +0 -10
- data/test/log_fragments/fragment_5.log +0 -24
- data/test/log_fragments/merb_1.log +0 -84
- data/test/merb_log_parser_test.rb +0 -39
- data/test/rails_log_parser_test.rb +0 -94
- data/test/record_inserter_test.rb +0 -45
data/spec/filter_spec.rb
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
|
3
|
+
require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
|
4
|
+
|
5
|
+
describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
|
6
|
+
include RequestLogAnalyzerSpecHelper
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
|
10
|
+
@filter.prepare
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should reject a request before the after date" do
|
14
|
+
@filter.filter(request(:timestamp => 20081212000000)).should be_nil
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should reject a request after the before date" do
|
18
|
+
@filter.filter(request(:timestamp => 20090303000000)).should be_nil
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should accept a request between the after and before dates" do
|
22
|
+
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
|
27
|
+
include RequestLogAnalyzerSpecHelper
|
28
|
+
|
29
|
+
before(:each) do
|
30
|
+
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))
|
31
|
+
@filter.prepare
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should accept a request before the after date" do
|
35
|
+
@filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
|
36
|
+
end
|
37
|
+
|
38
|
+
it "should reject a request after the before date" do
|
39
|
+
@filter.filter(request(:timestamp => 20090303000000)).should be_nil
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should accept a request between the after and before dates" do
|
43
|
+
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
|
48
|
+
include RequestLogAnalyzerSpecHelper
|
49
|
+
|
50
|
+
before(:each) do
|
51
|
+
@filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
|
52
|
+
@filter.prepare
|
53
|
+
end
|
54
|
+
|
55
|
+
it "should reject a request before the after date" do
|
56
|
+
@filter.filter(request(:timestamp => 20081212000000)).should be_nil
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should accept a request after the before date" do
|
60
|
+
@filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
|
61
|
+
end
|
62
|
+
|
63
|
+
it "should accept a request between the after and before dates" do
|
64
|
+
@filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
|
69
|
+
include RequestLogAnalyzerSpecHelper
|
70
|
+
|
71
|
+
before(:each) do
|
72
|
+
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
|
73
|
+
@filter.prepare
|
74
|
+
end
|
75
|
+
|
76
|
+
it "should reject a request if the field value does not match" do
|
77
|
+
@filter.filter(request(:test => 'not test')).should be_nil
|
78
|
+
end
|
79
|
+
|
80
|
+
it "should reject a request if the field name does not match" do
|
81
|
+
@filter.filter(request(:testing => 'test')).should be_nil
|
82
|
+
end
|
83
|
+
|
84
|
+
it "should accept a request if the both name and value match" do
|
85
|
+
@filter.filter(request(:test => 'test')).should_not be_nil
|
86
|
+
end
|
87
|
+
|
88
|
+
it "should accept a request if the value is not the first value" do
|
89
|
+
@filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
|
94
|
+
include RequestLogAnalyzerSpecHelper
|
95
|
+
|
96
|
+
before(:each) do
|
97
|
+
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
|
98
|
+
@filter.prepare
|
99
|
+
end
|
100
|
+
|
101
|
+
it "should accept a request if the field value does not match" do
|
102
|
+
@filter.filter(request(:test => 'not test')).should_not be_nil
|
103
|
+
end
|
104
|
+
|
105
|
+
it "should accept a request if the field name does not match" do
|
106
|
+
@filter.filter(request(:testing => 'test')).should_not be_nil
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should reject a request if the both name and value match" do
|
110
|
+
@filter.filter(request(:test => 'test')).should be_nil
|
111
|
+
end
|
112
|
+
|
113
|
+
it "should reject a request if the value is not the first value" do
|
114
|
+
@filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
|
119
|
+
include RequestLogAnalyzerSpecHelper
|
120
|
+
|
121
|
+
before(:each) do
|
122
|
+
@filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
|
123
|
+
@filter.prepare
|
124
|
+
end
|
125
|
+
|
126
|
+
it "should reject a request if the field value does not match" do
|
127
|
+
@filter.filter(request(:test => 'a working test')).should_not be_nil
|
128
|
+
end
|
129
|
+
|
130
|
+
it "should reject a request if the field name does not match" do
|
131
|
+
@filter.filter(request(:testing => 'test')).should be_nil
|
132
|
+
end
|
133
|
+
|
134
|
+
it "should accept a request if the value is not the first value" do
|
135
|
+
@filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
require 'request_log_analyzer/log_processor'
|
3
|
+
|
4
|
+
describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
|
5
|
+
|
6
|
+
include RequestLogAnalyzerSpecHelper
|
7
|
+
|
8
|
+
before(:each) do
|
9
|
+
@log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
|
10
|
+
@alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should keep a junk line if :keep_junk_lines is true" do
|
14
|
+
@alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
|
15
|
+
end
|
16
|
+
|
17
|
+
it "should remove a junk line" do
|
18
|
+
@log_anonymizer.anonymize_line("junk line\n").should be_empty
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should keep a teaser line intact" do
|
22
|
+
@log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should discard a teaser line if discard_teaser_line is true" do
|
26
|
+
@alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should keep a matching line intact if no anonymizing is declared" do
|
30
|
+
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
|
31
|
+
end
|
32
|
+
|
33
|
+
it "should anonymize values completely if requested" do
|
34
|
+
@alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should anonymize values slightly if requested" do
|
38
|
+
@alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
|
43
|
+
|
44
|
+
include RequestLogAnalyzerSpecHelper
|
45
|
+
|
46
|
+
before(:each) do
|
47
|
+
@log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
|
48
|
+
end
|
49
|
+
|
50
|
+
it "should remove a junk line" do
|
51
|
+
@log_stripper.strip_line("junk line\n").should be_empty
|
52
|
+
end
|
53
|
+
|
54
|
+
it "should keep a teaser line intact" do
|
55
|
+
@log_stripper.strip_line("processing 1234\n").should be_empty
|
56
|
+
end
|
57
|
+
end
|
data/tasks/rspec.rake
ADDED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: wvanbergen-request-log-analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Willem van Bergen
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2009-
|
13
|
+
date: 2009-01-12 00:00:00 -08:00
|
14
14
|
default_executable: request-log-analyzer
|
15
15
|
dependencies: []
|
16
16
|
|
@@ -18,57 +18,59 @@ description: Rails log analyzer's purpose is to find what actions are best candi
|
|
18
18
|
email: willem@vanbergen.org
|
19
19
|
executables:
|
20
20
|
- request-log-analyzer
|
21
|
-
- request-log-database
|
22
21
|
extensions: []
|
23
22
|
|
24
23
|
extra_rdoc_files: []
|
25
24
|
|
26
25
|
files:
|
26
|
+
- DESIGN
|
27
|
+
- HACKING
|
27
28
|
- LICENSE
|
28
29
|
- README.textile
|
29
30
|
- Rakefile
|
30
|
-
- TODO
|
31
31
|
- bin
|
32
32
|
- bin/request-log-analyzer
|
33
|
-
- bin/request-log-database
|
34
33
|
- lib
|
35
|
-
- lib/
|
36
|
-
- lib/
|
37
|
-
- lib/
|
38
|
-
- lib/
|
39
|
-
- lib/
|
40
|
-
- lib/command_line
|
41
|
-
- lib/command_line/arguments.rb
|
42
|
-
- lib/command_line/exceptions.rb
|
43
|
-
- lib/command_line/flag.rb
|
44
|
-
- lib/merb_analyzer
|
45
|
-
- lib/merb_analyzer/log_parser.rb
|
46
|
-
- lib/merb_analyzer/summarizer.rb
|
47
|
-
- lib/rails_analyzer
|
48
|
-
- lib/rails_analyzer/log_parser.rb
|
49
|
-
- lib/rails_analyzer/record_inserter.rb
|
50
|
-
- lib/rails_analyzer/summarizer.rb
|
51
|
-
- lib/rails_analyzer/virtual_mongrel.rb
|
34
|
+
- lib/cli
|
35
|
+
- lib/cli/bashcolorizer.rb
|
36
|
+
- lib/cli/command_line_arguments.rb
|
37
|
+
- lib/cli/progressbar.rb
|
38
|
+
- lib/request_log_analyzer
|
52
39
|
- lib/request_log_analyzer.rb
|
53
|
-
- lib/
|
54
|
-
- lib/
|
55
|
-
- lib/
|
56
|
-
- lib/
|
57
|
-
-
|
58
|
-
-
|
59
|
-
-
|
60
|
-
-
|
61
|
-
-
|
62
|
-
-
|
63
|
-
-
|
64
|
-
-
|
65
|
-
-
|
66
|
-
-
|
67
|
-
-
|
68
|
-
-
|
40
|
+
- lib/request_log_analyzer/aggregator
|
41
|
+
- lib/request_log_analyzer/aggregator/base.rb
|
42
|
+
- lib/request_log_analyzer/aggregator/database.rb
|
43
|
+
- lib/request_log_analyzer/aggregator/echo.rb
|
44
|
+
- lib/request_log_analyzer/aggregator/summarizer.rb
|
45
|
+
- lib/request_log_analyzer/controller.rb
|
46
|
+
- lib/request_log_analyzer/file_format
|
47
|
+
- lib/request_log_analyzer/file_format.rb
|
48
|
+
- lib/request_log_analyzer/file_format/merb.rb
|
49
|
+
- lib/request_log_analyzer/file_format/rails.rb
|
50
|
+
- lib/request_log_analyzer/filter
|
51
|
+
- lib/request_log_analyzer/filter/base.rb
|
52
|
+
- lib/request_log_analyzer/filter/field.rb
|
53
|
+
- lib/request_log_analyzer/filter/timespan.rb
|
54
|
+
- lib/request_log_analyzer/line_definition.rb
|
55
|
+
- lib/request_log_analyzer/log_parser.rb
|
56
|
+
- lib/request_log_analyzer/log_processor.rb
|
57
|
+
- lib/request_log_analyzer/request.rb
|
58
|
+
- lib/request_log_analyzer/source
|
59
|
+
- lib/request_log_analyzer/source/base.rb
|
60
|
+
- lib/request_log_analyzer/source/log_file.rb
|
61
|
+
- lib/request_log_analyzer/tracker
|
62
|
+
- lib/request_log_analyzer/tracker/base.rb
|
63
|
+
- lib/request_log_analyzer/tracker/category.rb
|
64
|
+
- lib/request_log_analyzer/tracker/duration.rb
|
65
|
+
- lib/request_log_analyzer/tracker/hourly_spread.rb
|
66
|
+
- lib/request_log_analyzer/tracker/timespan.rb
|
69
67
|
- spec
|
70
68
|
- spec/controller_spec.rb
|
71
69
|
- spec/database_inserter_spec.rb
|
70
|
+
- spec/file_format_spec.rb
|
71
|
+
- spec/file_formats
|
72
|
+
- spec/file_formats/spec_format.rb
|
73
|
+
- spec/filter_spec.rb
|
72
74
|
- spec/fixtures
|
73
75
|
- spec/fixtures/merb.log
|
74
76
|
- spec/fixtures/multiple_files_1.log
|
@@ -83,6 +85,7 @@ files:
|
|
83
85
|
- spec/fixtures/test_order.log
|
84
86
|
- spec/line_definition_spec.rb
|
85
87
|
- spec/log_parser_spec.rb
|
88
|
+
- spec/log_processor_spec.rb
|
86
89
|
- spec/merb_format_spec.rb
|
87
90
|
- spec/rails_format_spec.rb
|
88
91
|
- spec/request_spec.rb
|
@@ -91,19 +94,7 @@ files:
|
|
91
94
|
- tasks
|
92
95
|
- tasks/github-gem.rake
|
93
96
|
- tasks/request_log_analyzer.rake
|
94
|
-
- tasks/
|
95
|
-
- test
|
96
|
-
- test/base_summarizer_test.rb
|
97
|
-
- test/log_fragments
|
98
|
-
- test/log_fragments/fragment_1.log
|
99
|
-
- test/log_fragments/fragment_2.log
|
100
|
-
- test/log_fragments/fragment_3.log
|
101
|
-
- test/log_fragments/fragment_4.log
|
102
|
-
- test/log_fragments/fragment_5.log
|
103
|
-
- test/log_fragments/merb_1.log
|
104
|
-
- test/merb_log_parser_test.rb
|
105
|
-
- test/rails_log_parser_test.rb
|
106
|
-
- test/record_inserter_test.rb
|
97
|
+
- tasks/rspec.rake
|
107
98
|
has_rdoc: false
|
108
99
|
homepage: http://github.com/wvanbergen/request-log-analyzer/wikis
|
109
100
|
post_install_message:
|
@@ -131,7 +122,14 @@ signing_key:
|
|
131
122
|
specification_version: 2
|
132
123
|
summary: A command line tool to analyze Rails logs
|
133
124
|
test_files:
|
134
|
-
-
|
135
|
-
-
|
136
|
-
-
|
137
|
-
-
|
125
|
+
- spec/controller_spec.rb
|
126
|
+
- spec/database_inserter_spec.rb
|
127
|
+
- spec/file_format_spec.rb
|
128
|
+
- spec/filter_spec.rb
|
129
|
+
- spec/line_definition_spec.rb
|
130
|
+
- spec/log_parser_spec.rb
|
131
|
+
- spec/log_processor_spec.rb
|
132
|
+
- spec/merb_format_spec.rb
|
133
|
+
- spec/rails_format_spec.rb
|
134
|
+
- spec/request_spec.rb
|
135
|
+
- spec/summarizer_spec.rb
|
data/TODO
DELETED
@@ -1,58 +0,0 @@
|
|
1
|
-
TODO items for Rails-log-analyzer
|
2
|
-
=================================
|
3
|
-
Contact willem AT vanbergen DOT org if you want to help out with the development.
|
4
|
-
|
5
|
-
General:
|
6
|
-
- Add more tests / specs
|
7
|
-
|
8
|
-
Parameters:
|
9
|
-
- Add commandline parameters
|
10
|
-
--version, -v -> This should support github - last version checking
|
11
|
-
--help, -h -> Show usage
|
12
|
-
|
13
|
-
Datamining:
|
14
|
-
- Add query functionality for the resulting database file (interactive reports?)
|
15
|
-
- Link request processing line to request completed line (VirtualMongrel?)
|
16
|
-
- Fix the database inserter and make it more robust for future changes
|
17
|
-
|
18
|
-
Rails integration:
|
19
|
-
- Optionally use local or specific routes.rb file to parse URLs
|
20
|
-
|
21
|
-
Other:
|
22
|
-
- World domination
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
Datamining should look something like this:
|
28
|
-
|
29
|
-
> request-log-analyzer myapp.log --interactive
|
30
|
-
Request log analyzer builds a new database.
|
31
|
-
Columns come from the log_parser as the LOG_LINES store all the keys that they can detect.
|
32
|
-
Also add some extra columns like hashed_request_url etc.
|
33
|
-
|
34
|
-
Request log analyzer then parses the logfile for its individual requests using something like the
|
35
|
-
virtual mongrel (we need a new name for this, database_summarizer agregator? sheepdog?) combined with our
|
36
|
-
default log parser.
|
37
|
-
|
38
|
-
When this is done the user enters an interactive mode (like irb).
|
39
|
-
> Filters: None
|
40
|
-
> Total requests in database: 53232
|
41
|
-
> $
|
42
|
-
|
43
|
-
The user can add filters like this:
|
44
|
-
> $ FILTER SQL ["date > ?", Date.today-1]
|
45
|
-
|
46
|
-
The user will then see this:
|
47
|
-
> Filters:
|
48
|
-
> 1. ["date > ?", Date.today-1]
|
49
|
-
> Total requests: 2120
|
50
|
-
> $
|
51
|
-
|
52
|
-
At any point the user can destroy filters, show the raw requests or show reports
|
53
|
-
> $ REPORT ALL
|
54
|
-
|
55
|
-
The request remaining after the filter chain will then be processed through the summarizer and then trough the
|
56
|
-
output templates, generating reports specificly for the selected dataset.
|
57
|
-
Partials should also be possible
|
58
|
-
> $ REPORT TIMESPAN
|
data/bin/request-log-database
DELETED
@@ -1,81 +0,0 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
|
3
|
-
require File.dirname(__FILE__) + '/../lib/command_line/arguments'
|
4
|
-
require File.dirname(__FILE__) + '/../lib/base/log_parser'
|
5
|
-
require File.dirname(__FILE__) + '/../lib/base/record_inserter'
|
6
|
-
require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
|
7
|
-
require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
|
8
|
-
require File.dirname(__FILE__) + '/../lib/bashcolorizer'
|
9
|
-
require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
|
10
|
-
|
11
|
-
|
12
|
-
puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
|
13
|
-
|
14
|
-
begin
|
15
|
-
|
16
|
-
$arguments = CommandLine::Arguments.parse do |command_line|
|
17
|
-
command_line.switch(:guess_database_time, :g)
|
18
|
-
command_line.switch(:reset_database, :r)
|
19
|
-
command_line.flag(:database, :alias => :d, :required => false)
|
20
|
-
command_line.required_files = 1
|
21
|
-
end
|
22
|
-
|
23
|
-
rescue CommandLine::Error => e
|
24
|
-
puts "ARGUMENT ERROR: " + e.message
|
25
|
-
puts
|
26
|
-
puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
|
27
|
-
puts
|
28
|
-
puts "Options:"
|
29
|
-
puts " --database, -t: The database file to use"
|
30
|
-
puts " --reset-database, -r: Resets the database before inserting new records"
|
31
|
-
puts " --guess-database-time, -g: Guesses the database duration of requests"
|
32
|
-
puts
|
33
|
-
puts "Examples:"
|
34
|
-
puts " ./parsetodb.rb development.log"
|
35
|
-
puts " ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"
|
36
|
-
puts
|
37
|
-
|
38
|
-
exit(0)
|
39
|
-
end
|
40
|
-
|
41
|
-
log_files = $arguments.files
|
42
|
-
db_file = $arguments[:database] || log_files.first + '.db'
|
43
|
-
|
44
|
-
if $arguments[:reset_database] && File.exist?(db_file)
|
45
|
-
File.delete(db_file)
|
46
|
-
puts "Database file cleared."
|
47
|
-
end
|
48
|
-
|
49
|
-
records_inserted = 0
|
50
|
-
inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
|
51
|
-
log_files.each do |log_file|
|
52
|
-
|
53
|
-
puts "Processing all log lines from #{log_file}..."
|
54
|
-
parser = RailsAnalyzer::LogParser.new(log_file)
|
55
|
-
|
56
|
-
pbar = ProgressBar.new(green(log_file), File.size(log_file))
|
57
|
-
parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
|
58
|
-
|
59
|
-
parser.each do |request|
|
60
|
-
db.insert(request)
|
61
|
-
records_inserted += 1
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
if $arguments[:guess_database_time]
|
66
|
-
puts "Calculating database times..."
|
67
|
-
db.calculate_db_durations!
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
started = inserter.count(:started)
|
72
|
-
completed = inserter.count(:completed)
|
73
|
-
failed = inserter.count(:failed)
|
74
|
-
|
75
|
-
puts
|
76
|
-
puts "Inserted #{records_inserted} records from #{log_files.length} files."
|
77
|
-
puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
|
78
|
-
puts
|
79
|
-
puts "Requests started: #{started}"
|
80
|
-
puts "Requests completed: #{completed}"
|
81
|
-
puts "Requests failed: #{failed}"
|
data/lib/base/log_parser.rb
DELETED
@@ -1,78 +0,0 @@
|
|
1
|
-
module Base
|
2
|
-
# Parse a log file
|
3
|
-
class LogParser
|
4
|
-
|
5
|
-
LOG_LINES = {}
|
6
|
-
|
7
|
-
# LogParser initializer
|
8
|
-
# <tt>file</tt> The fileobject this LogParser wil operate on.
|
9
|
-
def initialize(file, options = {})
|
10
|
-
@file_name = file
|
11
|
-
@options = options
|
12
|
-
@file_size = File.size(@file_name)
|
13
|
-
|
14
|
-
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
15
|
-
end
|
16
|
-
|
17
|
-
def progress(&block)
|
18
|
-
@progress_handler = block
|
19
|
-
end
|
20
|
-
|
21
|
-
# Output a warning
|
22
|
-
# <tt>message</tt> The warning message (object)
|
23
|
-
def warn(message)
|
24
|
-
puts " -> " + message.to_s
|
25
|
-
end
|
26
|
-
|
27
|
-
def convert_value(value, type)
|
28
|
-
return case type
|
29
|
-
when :string; value.to_s
|
30
|
-
when :int; value.to_i
|
31
|
-
when :sec; value.to_f
|
32
|
-
when :msec; value.to_f / 1000
|
33
|
-
when :timestamp; value.to_s # TODO: fix me?
|
34
|
-
else
|
35
|
-
warn("Unkwn type encountered: #{type}")
|
36
|
-
value
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
# Finds a log line and then parses the information in the line.
|
41
|
-
# Yields a hash containing the information found.
|
42
|
-
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
43
|
-
# Yeilds a Hash when it encounters a chunk of information.
|
44
|
-
def each(*line_types, &block)
|
45
|
-
log_lines_hash = self.class::LOG_LINES
|
46
|
-
|
47
|
-
|
48
|
-
# parse everything by default
|
49
|
-
line_types = log_lines_hash.keys if line_types.empty?
|
50
|
-
|
51
|
-
File.open(@file_name) do |file|
|
52
|
-
|
53
|
-
file.each_line do |line|
|
54
|
-
|
55
|
-
@progress_handler.call(file.pos, @file_size) if @progress_handler
|
56
|
-
|
57
|
-
line_types.each do |line_type|
|
58
|
-
if log_lines_hash[line_type][:teaser] =~ line
|
59
|
-
if log_lines_hash[line_type][:regexp] =~ line
|
60
|
-
|
61
|
-
captures = $~.captures
|
62
|
-
request = { :type => line_type, :line => file.lineno }
|
63
|
-
log_lines_hash[line_type][:params].each_with_index do |paramhash, index|
|
64
|
-
paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
|
65
|
-
end
|
66
|
-
|
67
|
-
yield(request) if block_given?
|
68
|
-
else
|
69
|
-
warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
@progress_handler.call(:finished, @file_size) if @progress_handler
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|