wvanbergen-request-log-analyzer 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/{README → README.textile} +29 -36
- data/Rakefile +3 -70
- data/TODO +43 -8
- data/bin/request-log-analyzer +32 -99
- data/lib/base/summarizer.rb +14 -0
- data/lib/bashcolorizer.rb +1 -1
- data/lib/command_line/arguments.rb +15 -2
- data/lib/command_line/flag.rb +12 -0
- data/lib/rails_analyzer/summarizer.rb +12 -4
- data/lib/rails_analyzer/virtual_mongrel.rb +91 -0
- data/lib/request_log_analyzer/aggregator/base.rb +34 -0
- data/lib/request_log_analyzer/aggregator/database.rb +86 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +10 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +53 -0
- data/lib/request_log_analyzer/controller.rb +90 -0
- data/lib/request_log_analyzer/file_format/merb.rb +30 -0
- data/lib/request_log_analyzer/file_format/rails.rb +84 -0
- data/lib/request_log_analyzer/file_format.rb +91 -0
- data/lib/request_log_analyzer/log_parser.rb +122 -0
- data/lib/request_log_analyzer/request.rb +72 -0
- data/lib/request_log_analyzer.rb +5 -0
- data/output/blockers.rb +2 -4
- data/output/errors.rb +1 -2
- data/output/hourly_spread.rb +3 -3
- data/output/mean_db_time.rb +1 -2
- data/output/mean_rendering_time.rb +2 -3
- data/output/mean_time.rb +2 -3
- data/output/most_requested.rb +1 -2
- data/output/timespan.rb +10 -8
- data/output/total_db_time.rb +2 -3
- data/output/total_time.rb +2 -3
- data/output/usage.rb +3 -2
- data/spec/controller_spec.rb +33 -0
- data/spec/database_inserter_spec.rb +81 -0
- data/{test/log_fragments/merb_1.log → spec/fixtures/merb.log} +0 -0
- data/spec/fixtures/multiple_files_1.log +5 -0
- data/spec/fixtures/multiple_files_2.log +2 -0
- data/{test/log_fragments/fragment_1.log → spec/fixtures/rails_1x.log} +5 -5
- data/{test/log_fragments/fragment_3.log → spec/fixtures/rails_22.log} +2 -2
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/{test/log_fragments/fragment_2.log → spec/fixtures/syslog_1x.log} +0 -0
- data/spec/fixtures/test_file_format.log +11 -0
- data/spec/fixtures/test_language_combined.log +14 -0
- data/spec/fixtures/test_order.log +16 -0
- data/spec/line_definition_spec.rb +34 -0
- data/spec/log_parser_spec.rb +92 -0
- data/spec/merb_format_spec.rb +58 -0
- data/spec/rails_format_spec.rb +95 -0
- data/spec/request_spec.rb +76 -0
- data/spec/spec_helper.rb +49 -0
- data/spec/summarizer_spec.rb +109 -0
- data/tasks/github-gem.rake +177 -0
- data/tasks/request_log_analyzer.rake +10 -0
- data/tasks/rspec.rake +6 -0
- data/test/base_summarizer_test.rb +30 -0
- metadata +46 -22
- data/bin/request-log-database +0 -81
- data/lib/base/log_parser.rb +0 -78
- data/lib/base/record_inserter.rb +0 -139
- data/lib/merb_analyzer/log_parser.rb +0 -26
- data/lib/rails_analyzer/log_parser.rb +0 -35
- data/lib/rails_analyzer/record_inserter.rb +0 -39
- data/test/merb_log_parser_test.rb +0 -39
- data/test/rails_log_parser_test.rb +0 -95
- data/test/record_inserter_test.rb +0 -45
- data/test/tasks.rake +0 -8
@@ -0,0 +1,34 @@
|
|
1
|
+
module RequestLogAnalyzer::Aggregator
|
2
|
+
|
3
|
+
|
4
|
+
class Base
|
5
|
+
|
6
|
+
include RequestLogAnalyzer::FileFormat
|
7
|
+
|
8
|
+
attr_reader :options
|
9
|
+
|
10
|
+
def initialize(format, options = {})
|
11
|
+
self.register_file_format(format)
|
12
|
+
@options = options
|
13
|
+
end
|
14
|
+
|
15
|
+
def aggregate(request)
|
16
|
+
# implement me!
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
def prepare
|
21
|
+
end
|
22
|
+
|
23
|
+
def finalize
|
24
|
+
end
|
25
|
+
|
26
|
+
def warning(type, message, lineno)
|
27
|
+
end
|
28
|
+
|
29
|
+
def report(color = false)
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'activerecord'
|
3
|
+
|
4
|
+
module RequestLogAnalyzer::Aggregator
|
5
|
+
|
6
|
+
class Database < Base
|
7
|
+
|
8
|
+
attr_reader :request_id
|
9
|
+
|
10
|
+
def prepare
|
11
|
+
ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
|
12
|
+
|
13
|
+
File.unlink(options[:database]) if File.exist?(options[:database])
|
14
|
+
create_database_schema!
|
15
|
+
|
16
|
+
@request_id = 0
|
17
|
+
end
|
18
|
+
|
19
|
+
def aggregate(request)
|
20
|
+
@request_id += 1
|
21
|
+
|
22
|
+
request.lines.each do |line|
|
23
|
+
class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
|
24
|
+
|
25
|
+
attributes = line.reject { |k, v| [:line_type].include?(k) }
|
26
|
+
attributes[:request_id] = @request_id if options[:combined_requests]
|
27
|
+
file_format.const_get(class_name).create!(attributes)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
def warning(type, message, lineno)
|
33
|
+
file_format::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
|
34
|
+
end
|
35
|
+
|
36
|
+
protected
|
37
|
+
|
38
|
+
def create_database_table(name, definition)
|
39
|
+
ActiveRecord::Migration.suppress_messages do
|
40
|
+
ActiveRecord::Migration.create_table("#{name}_lines") do |t|
|
41
|
+
t.column(:request_id, :integer) #if options[:combined_requests]
|
42
|
+
t.column(:lineno, :integer)
|
43
|
+
definition.captures.each do |field|
|
44
|
+
# there is only on key/value pait in this hash
|
45
|
+
field.each { |key, capture_type| t.column(key, column_type(capture_type)) }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def create_warning_table_and_class
|
52
|
+
ActiveRecord::Migration.suppress_messages do
|
53
|
+
ActiveRecord::Migration.create_table("warnings") do |t|
|
54
|
+
t.string :warning_type, :limit => 30, :null => false
|
55
|
+
t.string :message
|
56
|
+
t.integer :lineno
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
file_format.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.const_defined?('Warning')
|
61
|
+
end
|
62
|
+
|
63
|
+
def create_activerecord_class(name, definition)
|
64
|
+
class_name = "#{name}_line".camelize
|
65
|
+
file_format.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.const_defined?(class_name)
|
66
|
+
end
|
67
|
+
|
68
|
+
def create_database_schema!
|
69
|
+
file_format.line_definitions.each do |name, definition|
|
70
|
+
create_database_table(name, definition)
|
71
|
+
create_activerecord_class(name, definition)
|
72
|
+
end
|
73
|
+
|
74
|
+
create_warning_table_and_class
|
75
|
+
end
|
76
|
+
|
77
|
+
def column_type(capture_type)
|
78
|
+
case capture_type
|
79
|
+
when :sec; :double
|
80
|
+
when :msec; :double
|
81
|
+
when :float; :double
|
82
|
+
else capture_type
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module RequestLogAnalyzer::Aggregator
|
2
|
+
|
3
|
+
class Summarizer < Base
|
4
|
+
|
5
|
+
attr_reader :buckets
|
6
|
+
|
7
|
+
def prepare
|
8
|
+
@buckets = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def aggregate(request)
|
12
|
+
if options[:combined_requests]
|
13
|
+
current_bucket_hash = @buckets
|
14
|
+
else
|
15
|
+
@buckets[request.line_type] ||= {}
|
16
|
+
current_bucket_hash = @buckets[request.line_type]
|
17
|
+
end
|
18
|
+
|
19
|
+
bucket_name = bucket_for(request)
|
20
|
+
current_bucket_hash[bucket_name] ||= default_bucket_content
|
21
|
+
update_bucket(current_bucket_hash[bucket_name], request)
|
22
|
+
end
|
23
|
+
|
24
|
+
def default_bucket_content
|
25
|
+
return { :count => 0 }
|
26
|
+
end
|
27
|
+
|
28
|
+
def update_bucket(bucket, request)
|
29
|
+
bucket[:count] += 1
|
30
|
+
end
|
31
|
+
|
32
|
+
def bucket_for(request)
|
33
|
+
'all'
|
34
|
+
end
|
35
|
+
|
36
|
+
def report(color = false)
|
37
|
+
if options[:combined_requests]
|
38
|
+
@buckets.each do |hash, values|
|
39
|
+
puts " #{hash[0..40].ljust(41)}: #{values[:count]}"
|
40
|
+
end
|
41
|
+
else
|
42
|
+
@buckets.each do |line_type, buckets|
|
43
|
+
puts "Line type #{line_type.inspect}:"
|
44
|
+
buckets.each do |hash, values|
|
45
|
+
puts " #{hash[0..40].ljust(41)}: #{values[:count]}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
|
3
|
+
class Controller
|
4
|
+
|
5
|
+
include RequestLogAnalyzer::FileFormat
|
6
|
+
|
7
|
+
attr_reader :aggregators
|
8
|
+
attr_reader :log_parser
|
9
|
+
attr_reader :sources
|
10
|
+
attr_reader :options
|
11
|
+
|
12
|
+
def self.build(arguments)
|
13
|
+
|
14
|
+
options = {}
|
15
|
+
options[:combined_requests] = arguments[:combined_requests]
|
16
|
+
options[:database] = arguments[:database] if arguments[:database]
|
17
|
+
|
18
|
+
# Create the controller with the correct file format
|
19
|
+
controller = Controller.new(arguments[:format].to_sym, options)
|
20
|
+
|
21
|
+
# register sources
|
22
|
+
arguments.files.each do |file|
|
23
|
+
controller << file if File.exist?(file)
|
24
|
+
end
|
25
|
+
|
26
|
+
# register aggregators
|
27
|
+
arguments[:aggregator].each { |agg| controller >> agg.to_sym }
|
28
|
+
|
29
|
+
|
30
|
+
# register the database
|
31
|
+
controller >> :database if arguments[:database] && !arguments[:aggregator].include?('database')
|
32
|
+
controller >> :summarizer if arguments[:aggregator].empty?
|
33
|
+
|
34
|
+
# register the echo aggregator in debug mode
|
35
|
+
controller >> :echo if arguments[:debug]
|
36
|
+
|
37
|
+
return controller
|
38
|
+
end
|
39
|
+
|
40
|
+
def initialize(format = :rails, options = {})
|
41
|
+
|
42
|
+
@options = options
|
43
|
+
@aggregators = []
|
44
|
+
@sources = []
|
45
|
+
|
46
|
+
register_file_format(format)
|
47
|
+
@log_parser = RequestLogAnalyzer::LogParser.new(file_format, @options)
|
48
|
+
|
49
|
+
@log_parser.on_warning do |type, message, lineno|
|
50
|
+
@aggregators.each { |agg| agg.warning(type, message, lineno) }
|
51
|
+
puts "WARNING #{type.inspect} on line #{lineno}: #{message}" unless options[:silent]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def add_aggregator(agg)
|
56
|
+
if agg.kind_of?(Symbol)
|
57
|
+
require File.dirname(__FILE__) + "/aggregator/#{agg}"
|
58
|
+
agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
|
59
|
+
end
|
60
|
+
|
61
|
+
@aggregators << agg.new(file_format, @options)
|
62
|
+
end
|
63
|
+
|
64
|
+
alias :>> :add_aggregator
|
65
|
+
|
66
|
+
def add_source(source)
|
67
|
+
@sources << source
|
68
|
+
end
|
69
|
+
|
70
|
+
alias :<< :add_source
|
71
|
+
|
72
|
+
def run!
|
73
|
+
|
74
|
+
@aggregators.each { |agg| agg.prepare }
|
75
|
+
|
76
|
+
handle_request = Proc.new { |request| @aggregators.each { |agg| agg.aggregate(request) } }
|
77
|
+
@sources.each do |source|
|
78
|
+
case source
|
79
|
+
when IO; @log_parser.parse_io(source, options, &handle_request)
|
80
|
+
when String; @log_parser.parse_file(source, options, &handle_request)
|
81
|
+
else; raise "Unknwon source provided"
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
@aggregators.each { |agg| agg.finalize }
|
86
|
+
@aggregators.each { |agg| agg.report(options[:colorize]) }
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module RequestLogAnalyzer::FileFormat::Merb
|
2
|
+
|
3
|
+
LINE_DEFINITIONS = {
|
4
|
+
|
5
|
+
# ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
|
6
|
+
:started => {
|
7
|
+
:header => true,
|
8
|
+
:teaser => /Started/,
|
9
|
+
:regexp => /Started request handling\:\ (.+)/,
|
10
|
+
:captures => [{:timestamp => :timestamp}]
|
11
|
+
},
|
12
|
+
|
13
|
+
# ~ Params: {"action"=>"create", "controller"=>"session"}
|
14
|
+
# ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
|
15
|
+
:params => {
|
16
|
+
:teaser => /Params/,
|
17
|
+
:regexp => /Params\:\ \{(.+)\}/,
|
18
|
+
:captures => [{:raw_hash => :string}]
|
19
|
+
},
|
20
|
+
|
21
|
+
# ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
|
22
|
+
:completed => {
|
23
|
+
:footer => true,
|
24
|
+
:teaser => /\{:dispatch_time/,
|
25
|
+
:regexp => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
|
26
|
+
:captures => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
module RequestLogAnalyzer::FileFormat::Rails
|
2
|
+
|
3
|
+
# Rails < 2.1 completed line example
|
4
|
+
# Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
|
5
|
+
RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
|
6
|
+
|
7
|
+
# Rails > 2.1 completed line example
|
8
|
+
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
9
|
+
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
|
10
|
+
|
11
|
+
|
12
|
+
LINE_DEFINITIONS = {
|
13
|
+
|
14
|
+
# Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
|
15
|
+
:started => {
|
16
|
+
:header => true,
|
17
|
+
:teaser => /Processing/,
|
18
|
+
:regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
|
19
|
+
:captures => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
|
20
|
+
},
|
21
|
+
|
22
|
+
# Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a998a2ff0 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a998af660@/home/floorplanner/beta/releases/20081224113708/app/controllers/page_controller.rb:14>}>] rendered_or_redirected.
|
23
|
+
# Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.
|
24
|
+
:cache_hit => {
|
25
|
+
:regexp => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/,
|
26
|
+
:captures => []
|
27
|
+
},
|
28
|
+
|
29
|
+
# RuntimeError (Cannot destroy employee): /app/models/employee.rb:198:in `before_destroy'
|
30
|
+
:failed => {
|
31
|
+
:footer => true,
|
32
|
+
:teaser => /Error/,
|
33
|
+
:regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
|
34
|
+
:captures => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
|
35
|
+
},
|
36
|
+
|
37
|
+
# Completed lines: see above
|
38
|
+
:completed => {
|
39
|
+
:footer => true,
|
40
|
+
:teaser => /Completed in /,
|
41
|
+
:regexp => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
|
42
|
+
:captures => [{:duration => :sec}, {:view => :sec}, {:db => :sec}, {:status => :integer}, {:url => :string}, # 2.1 variant
|
43
|
+
{:duration => :msec}, {:view => :msec}, {:db => :msec}, {:status => :integer}, {:url => :string}] # 2.2 variant
|
44
|
+
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
module Summarizer
|
49
|
+
|
50
|
+
def bucket_for(request)
|
51
|
+
if options[:combined_requests]
|
52
|
+
|
53
|
+
if request =~ :failed
|
54
|
+
"#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
55
|
+
else
|
56
|
+
format = request[:format] || 'html'
|
57
|
+
"#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
58
|
+
end
|
59
|
+
|
60
|
+
else
|
61
|
+
case request.line_type
|
62
|
+
when :started
|
63
|
+
format = request[:format] || 'html'
|
64
|
+
"#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
|
65
|
+
|
66
|
+
when :completed
|
67
|
+
url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
|
68
|
+
url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
|
69
|
+
|
70
|
+
url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
|
71
|
+
url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
|
72
|
+
url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
|
73
|
+
url
|
74
|
+
|
75
|
+
when :failed
|
76
|
+
request[:error]
|
77
|
+
else
|
78
|
+
raise "Cannot group this request: #{request.inspect}"
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
module FileFormat
|
3
|
+
|
4
|
+
def self.included(base)
|
5
|
+
base.send(:attr_reader, :file_format)
|
6
|
+
end
|
7
|
+
|
8
|
+
# Registers the correct language in the calling class (LogParser, Summarizer)
|
9
|
+
def register_file_format(format_module)
|
10
|
+
|
11
|
+
# Loads the module constant for built in file formats
|
12
|
+
if format_module.kind_of?(Symbol)
|
13
|
+
require "#{File.dirname(__FILE__)}/file_format/#{format_module}"
|
14
|
+
format_module = RequestLogAnalyzer::FileFormat.const_get(format_module.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
|
15
|
+
end
|
16
|
+
|
17
|
+
format_module.instance_eval do
|
18
|
+
def line_definitions
|
19
|
+
@line_definitions ||= self::LINE_DEFINITIONS.inject({}) do |hash, (name, definition)|
|
20
|
+
hash.merge!(name => LineDefinition.new(name, definition))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# checks whether the file format descriptor is valid
|
25
|
+
def valid?
|
26
|
+
@line_definitions.detect { |(name, ld)| ld.header } && @line_definitions.detect { |(name, ld)| ld.footer }
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# register language specific hooks in base class
|
31
|
+
hook_module = self.class.to_s.split('::').last
|
32
|
+
if format_module.const_defined?(hook_module) && format_module.const_get(hook_module).kind_of?(Module)
|
33
|
+
metaclass = (class << self; self; end)
|
34
|
+
metaclass.send(:include, format_module.const_get(hook_module))
|
35
|
+
end
|
36
|
+
|
37
|
+
@file_format = format_module
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
class LineDefinition
|
43
|
+
|
44
|
+
attr_reader :name
|
45
|
+
attr_accessor :teaser, :regexp, :captures
|
46
|
+
attr_accessor :header, :footer
|
47
|
+
|
48
|
+
def initialize(name, definition = {})
|
49
|
+
@name = name
|
50
|
+
definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
|
51
|
+
end
|
52
|
+
|
53
|
+
def convert_value(value, type)
|
54
|
+
case type
|
55
|
+
when :integer; value.to_i
|
56
|
+
when :float; value.to_f
|
57
|
+
when :decimal; value.to_f
|
58
|
+
when :symbol; value.to_sym
|
59
|
+
when :sec; value.to_f
|
60
|
+
when :msec; value.to_f / 1000
|
61
|
+
when :timestamp; value.to_s # TODO: fix me?
|
62
|
+
else value
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def matches(line, lineno = nil, parser = nil)
|
67
|
+
if @teaser.nil? || @teaser =~ line
|
68
|
+
if @regexp =~ line
|
69
|
+
request_info = { :line_type => name, :lineno => lineno }
|
70
|
+
captures_found = $~.captures
|
71
|
+
captures.each_with_index do |param, index|
|
72
|
+
unless captures_found[index].nil? || param == :ignore
|
73
|
+
# there is only one key/value pair in the param hash, each will only be called once
|
74
|
+
param.each { |key, type| request_info[key] = convert_value(captures_found[index], type) }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
return request_info
|
78
|
+
else
|
79
|
+
parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}") unless @teaser.nil? || parser.nil?
|
80
|
+
return false
|
81
|
+
end
|
82
|
+
else
|
83
|
+
return false
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
alias :=~ :matches
|
88
|
+
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
class LogParser
|
3
|
+
|
4
|
+
include RequestLogAnalyzer::FileFormat
|
5
|
+
|
6
|
+
attr_reader :options
|
7
|
+
attr_reader :current_request
|
8
|
+
attr_reader :parsed_lines
|
9
|
+
attr_reader :parsed_requests
|
10
|
+
|
11
|
+
def initialize(format, options = {})
|
12
|
+
@line_definitions = {}
|
13
|
+
@options = options
|
14
|
+
@parsed_lines = 0
|
15
|
+
@parsed_requests = 0
|
16
|
+
|
17
|
+
@current_io = nil
|
18
|
+
|
19
|
+
# install the file format module (see RequestLogAnalyzer::FileFormat)
|
20
|
+
# and register all the line definitions to the parser
|
21
|
+
self.register_file_format(format)
|
22
|
+
end
|
23
|
+
|
24
|
+
def parse_files(files, options = {}, &block)
|
25
|
+
files.each do |file|
|
26
|
+
parse_file(file, options, &block)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# Parses a file.
|
31
|
+
# Creates an IO stream for the provided file, and sends it to parse_io for further handling
|
32
|
+
def parse_file(file, options = {}, &block)
|
33
|
+
@progress_handler.call(:started, file) if @progress_handler
|
34
|
+
File.open(file, 'r') { |f| parse_io(f, options, &block) }
|
35
|
+
@progress_handler.call(:completed, file) if @progress_handler
|
36
|
+
end
|
37
|
+
|
38
|
+
# Finds a log line and then parses the information in the line.
|
39
|
+
# Yields a hash containing the information found.
|
40
|
+
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
41
|
+
# Yeilds a Hash when it encounters a chunk of information.
|
42
|
+
def parse_io(io, options = {}, &block)
|
43
|
+
|
44
|
+
# parse every line type by default
|
45
|
+
options[:line_types] ||= file_format.line_definitions.keys
|
46
|
+
|
47
|
+
# check whether all provided line types are valid
|
48
|
+
unknown = options[:line_types].reject { |line_type| file_format.line_definitions.has_key?(line_type) }
|
49
|
+
raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
|
50
|
+
|
51
|
+
@current_io = io
|
52
|
+
@current_io.each_line do |line|
|
53
|
+
|
54
|
+
@progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.lineno % 10 == 0
|
55
|
+
|
56
|
+
request_data = nil
|
57
|
+
if options[:line_types].detect { |line_type| request_data = file_format.line_definitions[line_type].matches(line, @current_io.lineno, self) }
|
58
|
+
@parsed_lines += 1
|
59
|
+
if @options[:combined_requests]
|
60
|
+
update_current_request(request_data, &block)
|
61
|
+
else
|
62
|
+
handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
|
63
|
+
@parsed_requests += 1
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
warn(:unclosed_request, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
69
|
+
|
70
|
+
@current_io = nil
|
71
|
+
end
|
72
|
+
|
73
|
+
# Pass a block to this function to install a progress handler
|
74
|
+
def on_progress(&block)
|
75
|
+
@progress_handler = block
|
76
|
+
end
|
77
|
+
|
78
|
+
def on_warning(&block)
|
79
|
+
@warning_handler = block
|
80
|
+
end
|
81
|
+
|
82
|
+
def warn(type, message)
|
83
|
+
@warning_handler.call(type, message, @current_io.lineno) if @warning_handler
|
84
|
+
end
|
85
|
+
|
86
|
+
protected
|
87
|
+
|
88
|
+
def update_current_request(request_data, &block)
|
89
|
+
if header_line?(request_data)
|
90
|
+
unless @current_request.nil?
|
91
|
+
warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
|
92
|
+
@current_request = nil
|
93
|
+
else
|
94
|
+
@current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
|
95
|
+
end
|
96
|
+
else
|
97
|
+
unless @current_request.nil?
|
98
|
+
@current_request << request_data
|
99
|
+
if footer_line?(request_data)
|
100
|
+
handle_request(@current_request, &block)
|
101
|
+
@current_request = nil
|
102
|
+
@parsed_requests += 1
|
103
|
+
end
|
104
|
+
else
|
105
|
+
warn(:no_current_request, "Parsebale line found outside of a request!")
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def handle_request(request, &block)
|
111
|
+
yield(request) if block_given?
|
112
|
+
end
|
113
|
+
|
114
|
+
def header_line?(hash)
|
115
|
+
file_format.line_definitions[hash[:line_type]].header
|
116
|
+
end
|
117
|
+
|
118
|
+
def footer_line?(hash)
|
119
|
+
file_format.line_definitions[hash[:line_type]].footer
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module RequestLogAnalyzer
|
2
|
+
class Request
|
3
|
+
|
4
|
+
include RequestLogAnalyzer::FileFormat
|
5
|
+
|
6
|
+
attr_reader :lines
|
7
|
+
|
8
|
+
def initialize(file_format)
|
9
|
+
@lines = []
|
10
|
+
register_file_format(file_format)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.create(file_format, *hashes)
|
14
|
+
request = self.new(file_format)
|
15
|
+
hashes.flatten.each { |hash| request << hash }
|
16
|
+
return request
|
17
|
+
end
|
18
|
+
|
19
|
+
# Adds another line to the request
|
20
|
+
def << (request_info_hash)
|
21
|
+
@lines << request_info_hash
|
22
|
+
end
|
23
|
+
|
24
|
+
# Checks whether the given line type was parsed from the log file for this request
|
25
|
+
def has_line_type?(line_type)
|
26
|
+
@lines.detect { |l| l[:line_type] == line_type.to_sym }
|
27
|
+
end
|
28
|
+
|
29
|
+
alias :=~ :has_line_type?
|
30
|
+
|
31
|
+
# Returns the value that was captured for the "field" of this request.
|
32
|
+
# This function will return the first value that was captured if the field
|
33
|
+
# was captured in multiple lines for a combined request.
|
34
|
+
def first(field)
|
35
|
+
@lines.detect { |fields| fields.has_key?(field) }[field] rescue nil
|
36
|
+
end
|
37
|
+
|
38
|
+
alias :[] :first
|
39
|
+
|
40
|
+
# Returns an array of all the "field" values that were captured for this request
|
41
|
+
def every(field)
|
42
|
+
@lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
|
43
|
+
end
|
44
|
+
|
45
|
+
def empty?
|
46
|
+
@lines.length == 0
|
47
|
+
end
|
48
|
+
|
49
|
+
def single_line?
|
50
|
+
@lines.length == 1
|
51
|
+
end
|
52
|
+
|
53
|
+
def combined?
|
54
|
+
@lines.length > 1
|
55
|
+
end
|
56
|
+
|
57
|
+
def completed?
|
58
|
+
header_found, footer_found = false, false
|
59
|
+
@lines.each do |line|
|
60
|
+
line_def = file_format.line_definitions[line[:line_type]]
|
61
|
+
header_found = true if line_def.header
|
62
|
+
footer_found = true if line_def.footer
|
63
|
+
end
|
64
|
+
header_found && footer_found
|
65
|
+
end
|
66
|
+
|
67
|
+
def line_type
|
68
|
+
raise "Not a single line request!" unless single_line?
|
69
|
+
lines.first[:line_type]
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,5 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
|
2
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/request'
|
3
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
|
4
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
|
5
|
+
require File.dirname(__FILE__) + '/request_log_analyzer/controller'
|