wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. data/DESIGN +14 -0
  2. data/HACKING +7 -0
  3. data/README.textile +9 -98
  4. data/Rakefile +2 -2
  5. data/bin/request-log-analyzer +1 -1
  6. data/lib/cli/bashcolorizer.rb +60 -0
  7. data/lib/cli/command_line_arguments.rb +301 -0
  8. data/lib/cli/progressbar.rb +236 -0
  9. data/lib/request_log_analyzer/aggregator/base.rb +51 -0
  10. data/lib/request_log_analyzer/aggregator/database.rb +97 -0
  11. data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
  12. data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
  13. data/lib/request_log_analyzer/controller.rb +206 -0
  14. data/lib/request_log_analyzer/file_format/merb.rb +33 -0
  15. data/lib/request_log_analyzer/file_format/rails.rb +119 -0
  16. data/lib/request_log_analyzer/file_format.rb +77 -0
  17. data/lib/request_log_analyzer/filter/base.rb +29 -0
  18. data/lib/request_log_analyzer/filter/field.rb +36 -0
  19. data/lib/request_log_analyzer/filter/timespan.rb +32 -0
  20. data/lib/request_log_analyzer/line_definition.rb +159 -0
  21. data/lib/request_log_analyzer/log_parser.rb +183 -0
  22. data/lib/request_log_analyzer/log_processor.rb +121 -0
  23. data/lib/request_log_analyzer/request.rb +115 -0
  24. data/lib/request_log_analyzer/source/base.rb +42 -0
  25. data/lib/request_log_analyzer/source/log_file.rb +180 -0
  26. data/lib/request_log_analyzer/tracker/base.rb +54 -0
  27. data/lib/request_log_analyzer/tracker/category.rb +71 -0
  28. data/lib/request_log_analyzer/tracker/duration.rb +81 -0
  29. data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
  30. data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
  31. data/spec/file_format_spec.rb +78 -0
  32. data/spec/file_formats/spec_format.rb +26 -0
  33. data/spec/filter_spec.rb +137 -0
  34. data/spec/log_processor_spec.rb +57 -0
  35. data/tasks/rspec.rake +6 -0
  36. metadata +53 -55
  37. data/TODO +0 -58
  38. data/bin/request-log-database +0 -81
  39. data/lib/base/log_parser.rb +0 -78
  40. data/lib/base/record_inserter.rb +0 -139
  41. data/lib/command_line/arguments.rb +0 -129
  42. data/lib/command_line/flag.rb +0 -51
  43. data/lib/merb_analyzer/log_parser.rb +0 -26
  44. data/lib/rails_analyzer/log_parser.rb +0 -35
  45. data/lib/rails_analyzer/record_inserter.rb +0 -39
  46. data/tasks/test.rake +0 -8
  47. data/test/log_fragments/fragment_1.log +0 -59
  48. data/test/log_fragments/fragment_2.log +0 -5
  49. data/test/log_fragments/fragment_3.log +0 -12
  50. data/test/log_fragments/fragment_4.log +0 -10
  51. data/test/log_fragments/fragment_5.log +0 -24
  52. data/test/log_fragments/merb_1.log +0 -84
  53. data/test/merb_log_parser_test.rb +0 -39
  54. data/test/rails_log_parser_test.rb +0 -94
  55. data/test/record_inserter_test.rb +0 -45
@@ -0,0 +1,137 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+ require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
3
+ require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
4
+
5
+ describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do
6
+ include RequestLogAnalyzerSpecHelper
7
+
8
+ before(:each) do
9
+ @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
10
+ @filter.prepare
11
+ end
12
+
13
+ it "should reject a request before the after date" do
14
+ @filter.filter(request(:timestamp => 20081212000000)).should be_nil
15
+ end
16
+
17
+ it "should reject a request after the before date" do
18
+ @filter.filter(request(:timestamp => 20090303000000)).should be_nil
19
+ end
20
+
21
+ it "should accept a request between the after and before dates" do
22
+ @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
23
+ end
24
+ end
25
+
26
+ describe RequestLogAnalyzer::Filter::Timespan, 'only before' do
27
+ include RequestLogAnalyzerSpecHelper
28
+
29
+ before(:each) do
30
+ @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))
31
+ @filter.prepare
32
+ end
33
+
34
+ it "should accept a request before the after date" do
35
+ @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
36
+ end
37
+
38
+ it "should reject a request after the before date" do
39
+ @filter.filter(request(:timestamp => 20090303000000)).should be_nil
40
+ end
41
+
42
+ it "should accept a request between the after and before dates" do
43
+ @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
44
+ end
45
+ end
46
+
47
+ describe RequestLogAnalyzer::Filter::Timespan, 'only after' do
48
+ include RequestLogAnalyzerSpecHelper
49
+
50
+ before(:each) do
51
+ @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
52
+ @filter.prepare
53
+ end
54
+
55
+ it "should reject a request before the after date" do
56
+ @filter.filter(request(:timestamp => 20081212000000)).should be_nil
57
+ end
58
+
59
+ it "should accept a request after the before date" do
60
+ @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
61
+ end
62
+
63
+ it "should accept a request between the after and before dates" do
64
+ @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
65
+ end
66
+ end
67
+
68
+ describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
69
+ include RequestLogAnalyzerSpecHelper
70
+
71
+ before(:each) do
72
+ @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
73
+ @filter.prepare
74
+ end
75
+
76
+ it "should reject a request if the field value does not match" do
77
+ @filter.filter(request(:test => 'not test')).should be_nil
78
+ end
79
+
80
+ it "should reject a request if the field name does not match" do
81
+ @filter.filter(request(:testing => 'test')).should be_nil
82
+ end
83
+
84
+ it "should accept a request if the both name and value match" do
85
+ @filter.filter(request(:test => 'test')).should_not be_nil
86
+ end
87
+
88
+ it "should accept a request if the value is not the first value" do
89
+ @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
90
+ end
91
+ end
92
+
93
+ describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
94
+ include RequestLogAnalyzerSpecHelper
95
+
96
+ before(:each) do
97
+ @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
98
+ @filter.prepare
99
+ end
100
+
101
+ it "should accept a request if the field value does not match" do
102
+ @filter.filter(request(:test => 'not test')).should_not be_nil
103
+ end
104
+
105
+ it "should accept a request if the field name does not match" do
106
+ @filter.filter(request(:testing => 'test')).should_not be_nil
107
+ end
108
+
109
+ it "should reject a request if the both name and value match" do
110
+ @filter.filter(request(:test => 'test')).should be_nil
111
+ end
112
+
113
+ it "should reject a request if the value is not the first value" do
114
+ @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
115
+ end
116
+ end
117
+
118
+ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
119
+ include RequestLogAnalyzerSpecHelper
120
+
121
+ before(:each) do
122
+ @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
123
+ @filter.prepare
124
+ end
125
+
126
+ it "should reject a request if the field value does not match" do
127
+ @filter.filter(request(:test => 'a working test')).should_not be_nil
128
+ end
129
+
130
+ it "should reject a request if the field name does not match" do
131
+ @filter.filter(request(:testing => 'test')).should be_nil
132
+ end
133
+
134
+ it "should accept a request if the value is not the first value" do
135
+ @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
136
+ end
137
+ end
@@ -0,0 +1,57 @@
1
+ require File.dirname(__FILE__) + '/spec_helper'
2
+ require 'request_log_analyzer/log_processor'
3
+
4
+ describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
5
+
6
+ include RequestLogAnalyzerSpecHelper
7
+
8
+ before(:each) do
9
+ @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
10
+ @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
11
+ end
12
+
13
+ it "should keep a junk line if :keep_junk_lines is true" do
14
+ @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
15
+ end
16
+
17
+ it "should remove a junk line" do
18
+ @log_anonymizer.anonymize_line("junk line\n").should be_empty
19
+ end
20
+
21
+ it "should keep a teaser line intact" do
22
+ @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
23
+ end
24
+
25
+ it "should discard a teaser line if discard_teaser_line is true" do
26
+ @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
27
+ end
28
+
29
+ it "should keep a matching line intact if no anonymizing is declared" do
30
+ @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
31
+ end
32
+
33
+ it "should anonymize values completely if requested" do
34
+ @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
35
+ end
36
+
37
+ it "should anonymize values slightly if requested" do
38
+ @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
39
+ end
40
+ end
41
+
42
+ describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
43
+
44
+ include RequestLogAnalyzerSpecHelper
45
+
46
+ before(:each) do
47
+ @log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
48
+ end
49
+
50
+ it "should remove a junk line" do
51
+ @log_stripper.strip_line("junk line\n").should be_empty
52
+ end
53
+
54
+ it "should keep a teaser line intact" do
55
+ @log_stripper.strip_line("processing 1234\n").should be_empty
56
+ end
57
+ end
data/tasks/rspec.rake ADDED
@@ -0,0 +1,6 @@
1
+ require 'spec/rake/spectask'
2
+
3
+ desc "Run all specs in spec directory (excluding plugin specs)"
4
+ Spec::Rake::SpecTask.new(:spec) do |t|
5
+ t.spec_files = FileList['spec/**/*_spec.rb']
6
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wvanbergen-request-log-analyzer
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2009-10-01 00:00:00 -07:00
13
+ date: 2009-01-12 00:00:00 -08:00
14
14
  default_executable: request-log-analyzer
15
15
  dependencies: []
16
16
 
@@ -18,57 +18,59 @@ description: Rails log analyzer's purpose is to find what actions are best candi
18
18
  email: willem@vanbergen.org
19
19
  executables:
20
20
  - request-log-analyzer
21
- - request-log-database
22
21
  extensions: []
23
22
 
24
23
  extra_rdoc_files: []
25
24
 
26
25
  files:
26
+ - DESIGN
27
+ - HACKING
27
28
  - LICENSE
28
29
  - README.textile
29
30
  - Rakefile
30
- - TODO
31
31
  - bin
32
32
  - bin/request-log-analyzer
33
- - bin/request-log-database
34
33
  - lib
35
- - lib/base
36
- - lib/base/log_parser.rb
37
- - lib/base/record_inserter.rb
38
- - lib/base/summarizer.rb
39
- - lib/bashcolorizer.rb
40
- - lib/command_line
41
- - lib/command_line/arguments.rb
42
- - lib/command_line/exceptions.rb
43
- - lib/command_line/flag.rb
44
- - lib/merb_analyzer
45
- - lib/merb_analyzer/log_parser.rb
46
- - lib/merb_analyzer/summarizer.rb
47
- - lib/rails_analyzer
48
- - lib/rails_analyzer/log_parser.rb
49
- - lib/rails_analyzer/record_inserter.rb
50
- - lib/rails_analyzer/summarizer.rb
51
- - lib/rails_analyzer/virtual_mongrel.rb
34
+ - lib/cli
35
+ - lib/cli/bashcolorizer.rb
36
+ - lib/cli/command_line_arguments.rb
37
+ - lib/cli/progressbar.rb
38
+ - lib/request_log_analyzer
52
39
  - lib/request_log_analyzer.rb
53
- - lib/ruby-progressbar
54
- - lib/ruby-progressbar/progressbar.en.rd
55
- - lib/ruby-progressbar/progressbar.ja.rd
56
- - lib/ruby-progressbar/progressbar.rb
57
- - output
58
- - output/blockers.rb
59
- - output/errors.rb
60
- - output/hourly_spread.rb
61
- - output/mean_db_time.rb
62
- - output/mean_rendering_time.rb
63
- - output/mean_time.rb
64
- - output/most_requested.rb
65
- - output/timespan.rb
66
- - output/total_db_time.rb
67
- - output/total_time.rb
68
- - output/usage.rb
40
+ - lib/request_log_analyzer/aggregator
41
+ - lib/request_log_analyzer/aggregator/base.rb
42
+ - lib/request_log_analyzer/aggregator/database.rb
43
+ - lib/request_log_analyzer/aggregator/echo.rb
44
+ - lib/request_log_analyzer/aggregator/summarizer.rb
45
+ - lib/request_log_analyzer/controller.rb
46
+ - lib/request_log_analyzer/file_format
47
+ - lib/request_log_analyzer/file_format.rb
48
+ - lib/request_log_analyzer/file_format/merb.rb
49
+ - lib/request_log_analyzer/file_format/rails.rb
50
+ - lib/request_log_analyzer/filter
51
+ - lib/request_log_analyzer/filter/base.rb
52
+ - lib/request_log_analyzer/filter/field.rb
53
+ - lib/request_log_analyzer/filter/timespan.rb
54
+ - lib/request_log_analyzer/line_definition.rb
55
+ - lib/request_log_analyzer/log_parser.rb
56
+ - lib/request_log_analyzer/log_processor.rb
57
+ - lib/request_log_analyzer/request.rb
58
+ - lib/request_log_analyzer/source
59
+ - lib/request_log_analyzer/source/base.rb
60
+ - lib/request_log_analyzer/source/log_file.rb
61
+ - lib/request_log_analyzer/tracker
62
+ - lib/request_log_analyzer/tracker/base.rb
63
+ - lib/request_log_analyzer/tracker/category.rb
64
+ - lib/request_log_analyzer/tracker/duration.rb
65
+ - lib/request_log_analyzer/tracker/hourly_spread.rb
66
+ - lib/request_log_analyzer/tracker/timespan.rb
69
67
  - spec
70
68
  - spec/controller_spec.rb
71
69
  - spec/database_inserter_spec.rb
70
+ - spec/file_format_spec.rb
71
+ - spec/file_formats
72
+ - spec/file_formats/spec_format.rb
73
+ - spec/filter_spec.rb
72
74
  - spec/fixtures
73
75
  - spec/fixtures/merb.log
74
76
  - spec/fixtures/multiple_files_1.log
@@ -83,6 +85,7 @@ files:
83
85
  - spec/fixtures/test_order.log
84
86
  - spec/line_definition_spec.rb
85
87
  - spec/log_parser_spec.rb
88
+ - spec/log_processor_spec.rb
86
89
  - spec/merb_format_spec.rb
87
90
  - spec/rails_format_spec.rb
88
91
  - spec/request_spec.rb
@@ -91,19 +94,7 @@ files:
91
94
  - tasks
92
95
  - tasks/github-gem.rake
93
96
  - tasks/request_log_analyzer.rake
94
- - tasks/test.rake
95
- - test
96
- - test/base_summarizer_test.rb
97
- - test/log_fragments
98
- - test/log_fragments/fragment_1.log
99
- - test/log_fragments/fragment_2.log
100
- - test/log_fragments/fragment_3.log
101
- - test/log_fragments/fragment_4.log
102
- - test/log_fragments/fragment_5.log
103
- - test/log_fragments/merb_1.log
104
- - test/merb_log_parser_test.rb
105
- - test/rails_log_parser_test.rb
106
- - test/record_inserter_test.rb
97
+ - tasks/rspec.rake
107
98
  has_rdoc: false
108
99
  homepage: http://github.com/wvanbergen/request-log-analyzer/wikis
109
100
  post_install_message:
@@ -131,7 +122,14 @@ signing_key:
131
122
  specification_version: 2
132
123
  summary: A command line tool to analyze Rails logs
133
124
  test_files:
134
- - test/base_summarizer_test.rb
135
- - test/merb_log_parser_test.rb
136
- - test/rails_log_parser_test.rb
137
- - test/record_inserter_test.rb
125
+ - spec/controller_spec.rb
126
+ - spec/database_inserter_spec.rb
127
+ - spec/file_format_spec.rb
128
+ - spec/filter_spec.rb
129
+ - spec/line_definition_spec.rb
130
+ - spec/log_parser_spec.rb
131
+ - spec/log_processor_spec.rb
132
+ - spec/merb_format_spec.rb
133
+ - spec/rails_format_spec.rb
134
+ - spec/request_spec.rb
135
+ - spec/summarizer_spec.rb
data/TODO DELETED
@@ -1,58 +0,0 @@
1
- TODO items for Rails-log-analyzer
2
- =================================
3
- Contact willem AT vanbergen DOT org if you want to help out with the development.
4
-
5
- General:
6
- - Add more tests / specs
7
-
8
- Parameters:
9
- - Add commandline parameters
10
- --version, -v -> This should support github - last version checking
11
- --help, -h -> Show usage
12
-
13
- Datamining:
14
- - Add query functionality for the resulting database file (interactive reports?)
15
- - Link request processing line to request completed line (VirtualMongrel?)
16
- - Fix the database inserter and make it more robust for future changes
17
-
18
- Rails integration:
19
- - Optionally use local or specific routes.rb file to parse URLs
20
-
21
- Other:
22
- - World domination
23
-
24
-
25
-
26
-
27
- Datamining should look something like this:
28
-
29
- > request-log-analyzer myapp.log --interactive
30
- Request log analyzer builds a new database.
31
- Columns come from the log_parser as the LOG_LINES store all the keys that they can detect.
32
- Also add some extra columns like hashed_request_url etc.
33
-
34
- Request log analyzer then parses the logfile for its individual requests using something like the
35
- virtual mongrel (we need a new name for this, database_summarizer agregator? sheepdog?) combined with our
36
- default log parser.
37
-
38
- When this is done the user enters an interactive mode (like irb).
39
- > Filters: None
40
- > Total requests in database: 53232
41
- > $
42
-
43
- The user can add filters like this:
44
- > $ FILTER SQL ["date > ?", Date.today-1]
45
-
46
- The user will then see this:
47
- > Filters:
48
- > 1. ["date > ?", Date.today-1]
49
- > Total requests: 2120
50
- > $
51
-
52
- At any point the user can destroy filters, show the raw requests or show reports
53
- > $ REPORT ALL
54
-
55
- The request remaining after the filter chain will then be processed through the summarizer and then trough the
56
- output templates, generating reports specificly for the selected dataset.
57
- Partials should also be possible
58
- > $ REPORT TIMESPAN
@@ -1,81 +0,0 @@
1
- #!/usr/bin/ruby
2
-
3
- require File.dirname(__FILE__) + '/../lib/command_line/arguments'
4
- require File.dirname(__FILE__) + '/../lib/base/log_parser'
5
- require File.dirname(__FILE__) + '/../lib/base/record_inserter'
6
- require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
7
- require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
8
- require File.dirname(__FILE__) + '/../lib/bashcolorizer'
9
- require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
10
-
11
-
12
- puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
13
-
14
- begin
15
-
16
- $arguments = CommandLine::Arguments.parse do |command_line|
17
- command_line.switch(:guess_database_time, :g)
18
- command_line.switch(:reset_database, :r)
19
- command_line.flag(:database, :alias => :d, :required => false)
20
- command_line.required_files = 1
21
- end
22
-
23
- rescue CommandLine::Error => e
24
- puts "ARGUMENT ERROR: " + e.message
25
- puts
26
- puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
27
- puts
28
- puts "Options:"
29
- puts " --database, -t: The database file to use"
30
- puts " --reset-database, -r: Resets the database before inserting new records"
31
- puts " --guess-database-time, -g: Guesses the database duration of requests"
32
- puts
33
- puts "Examples:"
34
- puts " ./parsetodb.rb development.log"
35
- puts " ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"
36
- puts
37
-
38
- exit(0)
39
- end
40
-
41
- log_files = $arguments.files
42
- db_file = $arguments[:database] || log_files.first + '.db'
43
-
44
- if $arguments[:reset_database] && File.exist?(db_file)
45
- File.delete(db_file)
46
- puts "Database file cleared."
47
- end
48
-
49
- records_inserted = 0
50
- inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
51
- log_files.each do |log_file|
52
-
53
- puts "Processing all log lines from #{log_file}..."
54
- parser = RailsAnalyzer::LogParser.new(log_file)
55
-
56
- pbar = ProgressBar.new(green(log_file), File.size(log_file))
57
- parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
58
-
59
- parser.each do |request|
60
- db.insert(request)
61
- records_inserted += 1
62
- end
63
- end
64
-
65
- if $arguments[:guess_database_time]
66
- puts "Calculating database times..."
67
- db.calculate_db_durations!
68
- end
69
- end
70
-
71
- started = inserter.count(:started)
72
- completed = inserter.count(:completed)
73
- failed = inserter.count(:failed)
74
-
75
- puts
76
- puts "Inserted #{records_inserted} records from #{log_files.length} files."
77
- puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
78
- puts
79
- puts "Requests started: #{started}"
80
- puts "Requests completed: #{completed}"
81
- puts "Requests failed: #{failed}"
@@ -1,78 +0,0 @@
1
- module Base
2
- # Parse a log file
3
- class LogParser
4
-
5
- LOG_LINES = {}
6
-
7
- # LogParser initializer
8
- # <tt>file</tt> The fileobject this LogParser wil operate on.
9
- def initialize(file, options = {})
10
- @file_name = file
11
- @options = options
12
- @file_size = File.size(@file_name)
13
-
14
- self.initialize_hook(options) if self.respond_to?(:initialize_hook)
15
- end
16
-
17
- def progress(&block)
18
- @progress_handler = block
19
- end
20
-
21
- # Output a warning
22
- # <tt>message</tt> The warning message (object)
23
- def warn(message)
24
- puts " -> " + message.to_s
25
- end
26
-
27
- def convert_value(value, type)
28
- return case type
29
- when :string; value.to_s
30
- when :int; value.to_i
31
- when :sec; value.to_f
32
- when :msec; value.to_f / 1000
33
- when :timestamp; value.to_s # TODO: fix me?
34
- else
35
- warn("Unkwn type encountered: #{type}")
36
- value
37
- end
38
- end
39
-
40
- # Finds a log line and then parses the information in the line.
41
- # Yields a hash containing the information found.
42
- # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
43
- # Yeilds a Hash when it encounters a chunk of information.
44
- def each(*line_types, &block)
45
- log_lines_hash = self.class::LOG_LINES
46
-
47
-
48
- # parse everything by default
49
- line_types = log_lines_hash.keys if line_types.empty?
50
-
51
- File.open(@file_name) do |file|
52
-
53
- file.each_line do |line|
54
-
55
- @progress_handler.call(file.pos, @file_size) if @progress_handler
56
-
57
- line_types.each do |line_type|
58
- if log_lines_hash[line_type][:teaser] =~ line
59
- if log_lines_hash[line_type][:regexp] =~ line
60
-
61
- captures = $~.captures
62
- request = { :type => line_type, :line => file.lineno }
63
- log_lines_hash[line_type][:params].each_with_index do |paramhash, index|
64
- paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
65
- end
66
-
67
- yield(request) if block_given?
68
- else
69
- warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
70
- end
71
- end
72
- end
73
- end
74
- @progress_handler.call(:finished, @file_size) if @progress_handler
75
- end
76
- end
77
- end
78
- end