wvanbergen-request-log-analyzer 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/{README → README.textile} +29 -36
  2. data/Rakefile +3 -70
  3. data/TODO +43 -8
  4. data/bin/request-log-analyzer +32 -99
  5. data/lib/base/summarizer.rb +14 -0
  6. data/lib/bashcolorizer.rb +1 -1
  7. data/lib/command_line/arguments.rb +15 -2
  8. data/lib/command_line/flag.rb +12 -0
  9. data/lib/rails_analyzer/summarizer.rb +12 -4
  10. data/lib/rails_analyzer/virtual_mongrel.rb +91 -0
  11. data/lib/request_log_analyzer/aggregator/base.rb +34 -0
  12. data/lib/request_log_analyzer/aggregator/database.rb +86 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +10 -0
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +53 -0
  15. data/lib/request_log_analyzer/controller.rb +90 -0
  16. data/lib/request_log_analyzer/file_format/merb.rb +30 -0
  17. data/lib/request_log_analyzer/file_format/rails.rb +84 -0
  18. data/lib/request_log_analyzer/file_format.rb +91 -0
  19. data/lib/request_log_analyzer/log_parser.rb +122 -0
  20. data/lib/request_log_analyzer/request.rb +72 -0
  21. data/lib/request_log_analyzer.rb +5 -0
  22. data/output/blockers.rb +2 -4
  23. data/output/errors.rb +1 -2
  24. data/output/hourly_spread.rb +3 -3
  25. data/output/mean_db_time.rb +1 -2
  26. data/output/mean_rendering_time.rb +2 -3
  27. data/output/mean_time.rb +2 -3
  28. data/output/most_requested.rb +1 -2
  29. data/output/timespan.rb +10 -8
  30. data/output/total_db_time.rb +2 -3
  31. data/output/total_time.rb +2 -3
  32. data/output/usage.rb +3 -2
  33. data/spec/controller_spec.rb +33 -0
  34. data/spec/database_inserter_spec.rb +81 -0
  35. data/{test/log_fragments/merb_1.log → spec/fixtures/merb.log} +0 -0
  36. data/spec/fixtures/multiple_files_1.log +5 -0
  37. data/spec/fixtures/multiple_files_2.log +2 -0
  38. data/{test/log_fragments/fragment_1.log → spec/fixtures/rails_1x.log} +5 -5
  39. data/{test/log_fragments/fragment_3.log → spec/fixtures/rails_22.log} +2 -2
  40. data/spec/fixtures/rails_22_cached.log +10 -0
  41. data/spec/fixtures/rails_unordered.log +24 -0
  42. data/{test/log_fragments/fragment_2.log → spec/fixtures/syslog_1x.log} +0 -0
  43. data/spec/fixtures/test_file_format.log +11 -0
  44. data/spec/fixtures/test_language_combined.log +14 -0
  45. data/spec/fixtures/test_order.log +16 -0
  46. data/spec/line_definition_spec.rb +34 -0
  47. data/spec/log_parser_spec.rb +92 -0
  48. data/spec/merb_format_spec.rb +58 -0
  49. data/spec/rails_format_spec.rb +95 -0
  50. data/spec/request_spec.rb +76 -0
  51. data/spec/spec_helper.rb +49 -0
  52. data/spec/summarizer_spec.rb +109 -0
  53. data/tasks/github-gem.rake +177 -0
  54. data/tasks/request_log_analyzer.rake +10 -0
  55. data/tasks/rspec.rake +6 -0
  56. data/test/base_summarizer_test.rb +30 -0
  57. metadata +46 -22
  58. data/bin/request-log-database +0 -81
  59. data/lib/base/log_parser.rb +0 -78
  60. data/lib/base/record_inserter.rb +0 -139
  61. data/lib/merb_analyzer/log_parser.rb +0 -26
  62. data/lib/rails_analyzer/log_parser.rb +0 -35
  63. data/lib/rails_analyzer/record_inserter.rb +0 -39
  64. data/test/merb_log_parser_test.rb +0 -39
  65. data/test/rails_log_parser_test.rb +0 -95
  66. data/test/record_inserter_test.rb +0 -45
  67. data/test/tasks.rake +0 -8
@@ -0,0 +1,34 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+
4
+ class Base
5
+
6
+ include RequestLogAnalyzer::FileFormat
7
+
8
+ attr_reader :options
9
+
10
+ def initialize(format, options = {})
11
+ self.register_file_format(format)
12
+ @options = options
13
+ end
14
+
15
+ def aggregate(request)
16
+ # implement me!
17
+ end
18
+
19
+
20
+ def prepare
21
+ end
22
+
23
+ def finalize
24
+ end
25
+
26
+ def warning(type, message, lineno)
27
+ end
28
+
29
+ def report(color = false)
30
+
31
+ end
32
+
33
+ end
34
+ end
@@ -0,0 +1,86 @@
1
+ require 'rubygems'
2
+ require 'activerecord'
3
+
4
+ module RequestLogAnalyzer::Aggregator
5
+
6
+ class Database < Base
7
+
8
+ attr_reader :request_id
9
+
10
+ def prepare
11
+ ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
12
+
13
+ File.unlink(options[:database]) if File.exist?(options[:database])
14
+ create_database_schema!
15
+
16
+ @request_id = 0
17
+ end
18
+
19
+ def aggregate(request)
20
+ @request_id += 1
21
+
22
+ request.lines.each do |line|
23
+ class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
24
+
25
+ attributes = line.reject { |k, v| [:line_type].include?(k) }
26
+ attributes[:request_id] = @request_id if options[:combined_requests]
27
+ file_format.const_get(class_name).create!(attributes)
28
+ end
29
+
30
+ end
31
+
32
+ def warning(type, message, lineno)
33
+ file_format::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
34
+ end
35
+
36
+ protected
37
+
38
+ def create_database_table(name, definition)
39
+ ActiveRecord::Migration.suppress_messages do
40
+ ActiveRecord::Migration.create_table("#{name}_lines") do |t|
41
+ t.column(:request_id, :integer) #if options[:combined_requests]
42
+ t.column(:lineno, :integer)
43
+ definition.captures.each do |field|
44
+ # there is only on key/value pait in this hash
45
+ field.each { |key, capture_type| t.column(key, column_type(capture_type)) }
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+ def create_warning_table_and_class
52
+ ActiveRecord::Migration.suppress_messages do
53
+ ActiveRecord::Migration.create_table("warnings") do |t|
54
+ t.string :warning_type, :limit => 30, :null => false
55
+ t.string :message
56
+ t.integer :lineno
57
+ end
58
+ end
59
+
60
+ file_format.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.const_defined?('Warning')
61
+ end
62
+
63
+ def create_activerecord_class(name, definition)
64
+ class_name = "#{name}_line".camelize
65
+ file_format.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.const_defined?(class_name)
66
+ end
67
+
68
+ def create_database_schema!
69
+ file_format.line_definitions.each do |name, definition|
70
+ create_database_table(name, definition)
71
+ create_activerecord_class(name, definition)
72
+ end
73
+
74
+ create_warning_table_and_class
75
+ end
76
+
77
+ def column_type(capture_type)
78
+ case capture_type
79
+ when :sec; :double
80
+ when :msec; :double
81
+ when :float; :double
82
+ else capture_type
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,10 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ class Echo < Base
4
+
5
+ def aggregate(request)
6
+ puts "\nRequest: " + request.inspect
7
+ end
8
+
9
+ end
10
+ end
@@ -0,0 +1,53 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ class Summarizer < Base
4
+
5
+ attr_reader :buckets
6
+
7
+ def prepare
8
+ @buckets = {}
9
+ end
10
+
11
+ def aggregate(request)
12
+ if options[:combined_requests]
13
+ current_bucket_hash = @buckets
14
+ else
15
+ @buckets[request.line_type] ||= {}
16
+ current_bucket_hash = @buckets[request.line_type]
17
+ end
18
+
19
+ bucket_name = bucket_for(request)
20
+ current_bucket_hash[bucket_name] ||= default_bucket_content
21
+ update_bucket(current_bucket_hash[bucket_name], request)
22
+ end
23
+
24
+ def default_bucket_content
25
+ return { :count => 0 }
26
+ end
27
+
28
+ def update_bucket(bucket, request)
29
+ bucket[:count] += 1
30
+ end
31
+
32
+ def bucket_for(request)
33
+ 'all'
34
+ end
35
+
36
+ def report(color = false)
37
+ if options[:combined_requests]
38
+ @buckets.each do |hash, values|
39
+ puts " #{hash[0..40].ljust(41)}: #{values[:count]}"
40
+ end
41
+ else
42
+ @buckets.each do |line_type, buckets|
43
+ puts "Line type #{line_type.inspect}:"
44
+ buckets.each do |hash, values|
45
+ puts " #{hash[0..40].ljust(41)}: #{values[:count]}"
46
+ end
47
+ end
48
+ end
49
+ end
50
+
51
+
52
+ end
53
+ end
@@ -0,0 +1,90 @@
1
+ module RequestLogAnalyzer
2
+
3
+ class Controller
4
+
5
+ include RequestLogAnalyzer::FileFormat
6
+
7
+ attr_reader :aggregators
8
+ attr_reader :log_parser
9
+ attr_reader :sources
10
+ attr_reader :options
11
+
12
+ def self.build(arguments)
13
+
14
+ options = {}
15
+ options[:combined_requests] = arguments[:combined_requests]
16
+ options[:database] = arguments[:database] if arguments[:database]
17
+
18
+ # Create the controller with the correct file format
19
+ controller = Controller.new(arguments[:format].to_sym, options)
20
+
21
+ # register sources
22
+ arguments.files.each do |file|
23
+ controller << file if File.exist?(file)
24
+ end
25
+
26
+ # register aggregators
27
+ arguments[:aggregator].each { |agg| controller >> agg.to_sym }
28
+
29
+
30
+ # register the database
31
+ controller >> :database if arguments[:database] && !arguments[:aggregator].include?('database')
32
+ controller >> :summarizer if arguments[:aggregator].empty?
33
+
34
+ # register the echo aggregator in debug mode
35
+ controller >> :echo if arguments[:debug]
36
+
37
+ return controller
38
+ end
39
+
40
+ def initialize(format = :rails, options = {})
41
+
42
+ @options = options
43
+ @aggregators = []
44
+ @sources = []
45
+
46
+ register_file_format(format)
47
+ @log_parser = RequestLogAnalyzer::LogParser.new(file_format, @options)
48
+
49
+ @log_parser.on_warning do |type, message, lineno|
50
+ @aggregators.each { |agg| agg.warning(type, message, lineno) }
51
+ puts "WARNING #{type.inspect} on line #{lineno}: #{message}" unless options[:silent]
52
+ end
53
+ end
54
+
55
+ def add_aggregator(agg)
56
+ if agg.kind_of?(Symbol)
57
+ require File.dirname(__FILE__) + "/aggregator/#{agg}"
58
+ agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
59
+ end
60
+
61
+ @aggregators << agg.new(file_format, @options)
62
+ end
63
+
64
+ alias :>> :add_aggregator
65
+
66
+ def add_source(source)
67
+ @sources << source
68
+ end
69
+
70
+ alias :<< :add_source
71
+
72
+ def run!
73
+
74
+ @aggregators.each { |agg| agg.prepare }
75
+
76
+ handle_request = Proc.new { |request| @aggregators.each { |agg| agg.aggregate(request) } }
77
+ @sources.each do |source|
78
+ case source
79
+ when IO; @log_parser.parse_io(source, options, &handle_request)
80
+ when String; @log_parser.parse_file(source, options, &handle_request)
81
+ else; raise "Unknwon source provided"
82
+ end
83
+ end
84
+
85
+ @aggregators.each { |agg| agg.finalize }
86
+ @aggregators.each { |agg| agg.report(options[:colorize]) }
87
+ end
88
+
89
+ end
90
+ end
@@ -0,0 +1,30 @@
1
+ module RequestLogAnalyzer::FileFormat::Merb
2
+
3
+ LINE_DEFINITIONS = {
4
+
5
+ # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
6
+ :started => {
7
+ :header => true,
8
+ :teaser => /Started/,
9
+ :regexp => /Started request handling\:\ (.+)/,
10
+ :captures => [{:timestamp => :timestamp}]
11
+ },
12
+
13
+ # ~ Params: {"action"=>"create", "controller"=>"session"}
14
+ # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
15
+ :params => {
16
+ :teaser => /Params/,
17
+ :regexp => /Params\:\ \{(.+)\}/,
18
+ :captures => [{:raw_hash => :string}]
19
+ },
20
+
21
+ # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
22
+ :completed => {
23
+ :footer => true,
24
+ :teaser => /\{:dispatch_time/,
25
+ :regexp => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
26
+ :captures => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
27
+ }
28
+ }
29
+
30
+ end
@@ -0,0 +1,84 @@
1
+ module RequestLogAnalyzer::FileFormat::Rails
2
+
3
+ # Rails < 2.1 completed line example
4
+ # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
5
+ RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
6
+
7
+ # Rails > 2.1 completed line example
8
+ # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
9
+ RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
10
+
11
+
12
+ LINE_DEFINITIONS = {
13
+
14
+ # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
15
+ :started => {
16
+ :header => true,
17
+ :teaser => /Processing/,
18
+ :regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
19
+ :captures => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
20
+ },
21
+
22
+ # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a998a2ff0 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a998af660@/home/floorplanner/beta/releases/20081224113708/app/controllers/page_controller.rb:14>}>] rendered_or_redirected.
23
+ # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
24
+ :cache_hit => {
25
+ :regexp => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/,
26
+ :captures => []
27
+ },
28
+
29
+ # RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy'
30
+ :failed => {
31
+ :footer => true,
32
+ :teaser => /Error/,
33
+ :regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
34
+ :captures => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
35
+ },
36
+
37
+ # Completed lines: see above
38
+ :completed => {
39
+ :footer => true,
40
+ :teaser => /Completed in /,
41
+ :regexp => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
42
+ :captures => [{:duration => :sec}, {:view => :sec}, {:db => :sec}, {:status => :integer}, {:url => :string}, # 2.1 variant
43
+ {:duration => :msec}, {:view => :msec}, {:db => :msec}, {:status => :integer}, {:url => :string}] # 2.2 variant
44
+
45
+ }
46
+ }
47
+
48
+ module Summarizer
49
+
50
+ def bucket_for(request)
51
+ if options[:combined_requests]
52
+
53
+ if request =~ :failed
54
+ "#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
55
+ else
56
+ format = request[:format] || 'html'
57
+ "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
58
+ end
59
+
60
+ else
61
+ case request.line_type
62
+ when :started
63
+ format = request[:format] || 'html'
64
+ "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
65
+
66
+ when :completed
67
+ url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
68
+ url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
69
+
70
+ url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
71
+ url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
72
+ url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
73
+ url
74
+
75
+ when :failed
76
+ request[:error]
77
+ else
78
+ raise "Cannot group this request: #{request.inspect}"
79
+ end
80
+ end
81
+ end
82
+ end
83
+
84
+ end
@@ -0,0 +1,91 @@
1
+ module RequestLogAnalyzer
2
+ module FileFormat
3
+
4
+ def self.included(base)
5
+ base.send(:attr_reader, :file_format)
6
+ end
7
+
8
+ # Registers the correct language in the calling class (LogParser, Summarizer)
9
+ def register_file_format(format_module)
10
+
11
+ # Loads the module constant for built in file formats
12
+ if format_module.kind_of?(Symbol)
13
+ require "#{File.dirname(__FILE__)}/file_format/#{format_module}"
14
+ format_module = RequestLogAnalyzer::FileFormat.const_get(format_module.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
15
+ end
16
+
17
+ format_module.instance_eval do
18
+ def line_definitions
19
+ @line_definitions ||= self::LINE_DEFINITIONS.inject({}) do |hash, (name, definition)|
20
+ hash.merge!(name => LineDefinition.new(name, definition))
21
+ end
22
+ end
23
+
24
+ # checks whether the file format descriptor is valid
25
+ def valid?
26
+ @line_definitions.detect { |(name, ld)| ld.header } && @line_definitions.detect { |(name, ld)| ld.footer }
27
+ end
28
+ end
29
+
30
+ # register language specific hooks in base class
31
+ hook_module = self.class.to_s.split('::').last
32
+ if format_module.const_defined?(hook_module) && format_module.const_get(hook_module).kind_of?(Module)
33
+ metaclass = (class << self; self; end)
34
+ metaclass.send(:include, format_module.const_get(hook_module))
35
+ end
36
+
37
+ @file_format = format_module
38
+ end
39
+
40
+
41
+
42
+ class LineDefinition
43
+
44
+ attr_reader :name
45
+ attr_accessor :teaser, :regexp, :captures
46
+ attr_accessor :header, :footer
47
+
48
+ def initialize(name, definition = {})
49
+ @name = name
50
+ definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
51
+ end
52
+
53
+ def convert_value(value, type)
54
+ case type
55
+ when :integer; value.to_i
56
+ when :float; value.to_f
57
+ when :decimal; value.to_f
58
+ when :symbol; value.to_sym
59
+ when :sec; value.to_f
60
+ when :msec; value.to_f / 1000
61
+ when :timestamp; value.to_s # TODO: fix me?
62
+ else value
63
+ end
64
+ end
65
+
66
+ def matches(line, lineno = nil, parser = nil)
67
+ if @teaser.nil? || @teaser =~ line
68
+ if @regexp =~ line
69
+ request_info = { :line_type => name, :lineno => lineno }
70
+ captures_found = $~.captures
71
+ captures.each_with_index do |param, index|
72
+ unless captures_found[index].nil? || param == :ignore
73
+ # there is only one key/value pair in the param hash, each will only be called once
74
+ param.each { |key, type| request_info[key] = convert_value(captures_found[index], type) }
75
+ end
76
+ end
77
+ return request_info
78
+ else
79
+ parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}") unless @teaser.nil? || parser.nil?
80
+ return false
81
+ end
82
+ else
83
+ return false
84
+ end
85
+ end
86
+
87
+ alias :=~ :matches
88
+
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,122 @@
1
+ module RequestLogAnalyzer
2
+ class LogParser
3
+
4
+ include RequestLogAnalyzer::FileFormat
5
+
6
+ attr_reader :options
7
+ attr_reader :current_request
8
+ attr_reader :parsed_lines
9
+ attr_reader :parsed_requests
10
+
11
+ def initialize(format, options = {})
12
+ @line_definitions = {}
13
+ @options = options
14
+ @parsed_lines = 0
15
+ @parsed_requests = 0
16
+
17
+ @current_io = nil
18
+
19
+ # install the file format module (see RequestLogAnalyzer::FileFormat)
20
+ # and register all the line definitions to the parser
21
+ self.register_file_format(format)
22
+ end
23
+
24
+ def parse_files(files, options = {}, &block)
25
+ files.each do |file|
26
+ parse_file(file, options, &block)
27
+ end
28
+ end
29
+
30
+ # Parses a file.
31
+ # Creates an IO stream for the provided file, and sends it to parse_io for further handling
32
+ def parse_file(file, options = {}, &block)
33
+ @progress_handler.call(:started, file) if @progress_handler
34
+ File.open(file, 'r') { |f| parse_io(f, options, &block) }
35
+ @progress_handler.call(:completed, file) if @progress_handler
36
+ end
37
+
38
+ # Finds a log line and then parses the information in the line.
39
+ # Yields a hash containing the information found.
40
+ # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
41
+ # Yeilds a Hash when it encounters a chunk of information.
42
+ def parse_io(io, options = {}, &block)
43
+
44
+ # parse every line type by default
45
+ options[:line_types] ||= file_format.line_definitions.keys
46
+
47
+ # check whether all provided line types are valid
48
+ unknown = options[:line_types].reject { |line_type| file_format.line_definitions.has_key?(line_type) }
49
+ raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
50
+
51
+ @current_io = io
52
+ @current_io.each_line do |line|
53
+
54
+ @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.lineno % 10 == 0
55
+
56
+ request_data = nil
57
+ if options[:line_types].detect { |line_type| request_data = file_format.line_definitions[line_type].matches(line, @current_io.lineno, self) }
58
+ @parsed_lines += 1
59
+ if @options[:combined_requests]
60
+ update_current_request(request_data, &block)
61
+ else
62
+ handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
63
+ @parsed_requests += 1
64
+ end
65
+ end
66
+ end
67
+
68
+ warn(:unclosed_request, "End of file reached, but last request was not completed!") unless @current_request.nil?
69
+
70
+ @current_io = nil
71
+ end
72
+
73
+ # Pass a block to this function to install a progress handler
74
+ def on_progress(&block)
75
+ @progress_handler = block
76
+ end
77
+
78
+ def on_warning(&block)
79
+ @warning_handler = block
80
+ end
81
+
82
+ def warn(type, message)
83
+ @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
84
+ end
85
+
86
+ protected
87
+
88
+ def update_current_request(request_data, &block)
89
+ if header_line?(request_data)
90
+ unless @current_request.nil?
91
+ warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
92
+ @current_request = nil
93
+ else
94
+ @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
95
+ end
96
+ else
97
+ unless @current_request.nil?
98
+ @current_request << request_data
99
+ if footer_line?(request_data)
100
+ handle_request(@current_request, &block)
101
+ @current_request = nil
102
+ @parsed_requests += 1
103
+ end
104
+ else
105
+ warn(:no_current_request, "Parsebale line found outside of a request!")
106
+ end
107
+ end
108
+ end
109
+
110
+ def handle_request(request, &block)
111
+ yield(request) if block_given?
112
+ end
113
+
114
+ def header_line?(hash)
115
+ file_format.line_definitions[hash[:line_type]].header
116
+ end
117
+
118
+ def footer_line?(hash)
119
+ file_format.line_definitions[hash[:line_type]].footer
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,72 @@
1
+ module RequestLogAnalyzer
2
+ class Request
3
+
4
+ include RequestLogAnalyzer::FileFormat
5
+
6
+ attr_reader :lines
7
+
8
+ def initialize(file_format)
9
+ @lines = []
10
+ register_file_format(file_format)
11
+ end
12
+
13
+ def self.create(file_format, *hashes)
14
+ request = self.new(file_format)
15
+ hashes.flatten.each { |hash| request << hash }
16
+ return request
17
+ end
18
+
19
+ # Adds another line to the request
20
+ def << (request_info_hash)
21
+ @lines << request_info_hash
22
+ end
23
+
24
+ # Checks whether the given line type was parsed from the log file for this request
25
+ def has_line_type?(line_type)
26
+ @lines.detect { |l| l[:line_type] == line_type.to_sym }
27
+ end
28
+
29
+ alias :=~ :has_line_type?
30
+
31
+ # Returns the value that was captured for the "field" of this request.
32
+ # This function will return the first value that was captured if the field
33
+ # was captured in multiple lines for a combined request.
34
+ def first(field)
35
+ @lines.detect { |fields| fields.has_key?(field) }[field] rescue nil
36
+ end
37
+
38
+ alias :[] :first
39
+
40
+ # Returns an array of all the "field" values that were captured for this request
41
+ def every(field)
42
+ @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
43
+ end
44
+
45
+ def empty?
46
+ @lines.length == 0
47
+ end
48
+
49
+ def single_line?
50
+ @lines.length == 1
51
+ end
52
+
53
+ def combined?
54
+ @lines.length > 1
55
+ end
56
+
57
+ def completed?
58
+ header_found, footer_found = false, false
59
+ @lines.each do |line|
60
+ line_def = file_format.line_definitions[line[:line_type]]
61
+ header_found = true if line_def.header
62
+ footer_found = true if line_def.footer
63
+ end
64
+ header_found && footer_found
65
+ end
66
+
67
+ def line_type
68
+ raise "Not a single line request!" unless single_line?
69
+ lines.first[:line_type]
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,5 @@
1
+ require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
2
+ require File.dirname(__FILE__) + '/request_log_analyzer/request'
3
+ require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
4
+ require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
5
+ require File.dirname(__FILE__) + '/request_log_analyzer/controller'