wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. data/DESIGN +14 -0
  2. data/HACKING +7 -0
  3. data/README.textile +9 -98
  4. data/Rakefile +2 -2
  5. data/bin/request-log-analyzer +1 -1
  6. data/lib/cli/bashcolorizer.rb +60 -0
  7. data/lib/cli/command_line_arguments.rb +301 -0
  8. data/lib/cli/progressbar.rb +236 -0
  9. data/lib/request_log_analyzer/aggregator/base.rb +51 -0
  10. data/lib/request_log_analyzer/aggregator/database.rb +97 -0
  11. data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
  12. data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
  13. data/lib/request_log_analyzer/controller.rb +206 -0
  14. data/lib/request_log_analyzer/file_format/merb.rb +33 -0
  15. data/lib/request_log_analyzer/file_format/rails.rb +119 -0
  16. data/lib/request_log_analyzer/file_format.rb +77 -0
  17. data/lib/request_log_analyzer/filter/base.rb +29 -0
  18. data/lib/request_log_analyzer/filter/field.rb +36 -0
  19. data/lib/request_log_analyzer/filter/timespan.rb +32 -0
  20. data/lib/request_log_analyzer/line_definition.rb +159 -0
  21. data/lib/request_log_analyzer/log_parser.rb +183 -0
  22. data/lib/request_log_analyzer/log_processor.rb +121 -0
  23. data/lib/request_log_analyzer/request.rb +115 -0
  24. data/lib/request_log_analyzer/source/base.rb +42 -0
  25. data/lib/request_log_analyzer/source/log_file.rb +180 -0
  26. data/lib/request_log_analyzer/tracker/base.rb +54 -0
  27. data/lib/request_log_analyzer/tracker/category.rb +71 -0
  28. data/lib/request_log_analyzer/tracker/duration.rb +81 -0
  29. data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
  30. data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
  31. data/spec/file_format_spec.rb +78 -0
  32. data/spec/file_formats/spec_format.rb +26 -0
  33. data/spec/filter_spec.rb +137 -0
  34. data/spec/log_processor_spec.rb +57 -0
  35. data/tasks/rspec.rake +6 -0
  36. metadata +53 -55
  37. data/TODO +0 -58
  38. data/bin/request-log-database +0 -81
  39. data/lib/base/log_parser.rb +0 -78
  40. data/lib/base/record_inserter.rb +0 -139
  41. data/lib/command_line/arguments.rb +0 -129
  42. data/lib/command_line/flag.rb +0 -51
  43. data/lib/merb_analyzer/log_parser.rb +0 -26
  44. data/lib/rails_analyzer/log_parser.rb +0 -35
  45. data/lib/rails_analyzer/record_inserter.rb +0 -39
  46. data/tasks/test.rake +0 -8
  47. data/test/log_fragments/fragment_1.log +0 -59
  48. data/test/log_fragments/fragment_2.log +0 -5
  49. data/test/log_fragments/fragment_3.log +0 -12
  50. data/test/log_fragments/fragment_4.log +0 -10
  51. data/test/log_fragments/fragment_5.log +0 -24
  52. data/test/log_fragments/merb_1.log +0 -84
  53. data/test/merb_log_parser_test.rb +0 -39
  54. data/test/rails_log_parser_test.rb +0 -94
  55. data/test/record_inserter_test.rb +0 -45
@@ -0,0 +1,236 @@
1
+ #
2
+ # Ruby/ProgressBar - a text progress bar library
3
+ #
4
+ # Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms
9
+ # of Ruby's license.
10
+ #
11
+
12
+ class ProgressBar
13
+ VERSION = "0.9"
14
+
15
+ def initialize (title, total, out = STDERR)
16
+ @title = title
17
+ @total = total
18
+ @out = out
19
+ @terminal_width = 80
20
+ @bar_mark = '='
21
+ @current = 0
22
+ @previous = 0
23
+ @finished_p = false
24
+ @start_time = Time.now
25
+ @previous_time = @start_time
26
+ @title_width = 24
27
+ @format = "%-#{@title_width}s %3d%% %s %s"
28
+ @format_arguments = [:title, :percentage, :bar, :stat]
29
+ clear
30
+ show
31
+ end
32
+ attr_reader :title
33
+ attr_reader :current
34
+ attr_reader :total
35
+ attr_accessor :start_time
36
+
37
+ private
38
+ def fmt_bar
39
+ bar_width = do_percentage * @terminal_width / 100
40
+ sprintf("[%s%s]",
41
+ @bar_mark * bar_width,
42
+ " " * (@terminal_width - bar_width))
43
+ end
44
+
45
+ def fmt_percentage
46
+ do_percentage
47
+ end
48
+
49
+ def fmt_stat
50
+ if @finished_p then elapsed else eta end
51
+ end
52
+
53
+ def fmt_stat_for_file_transfer
54
+ if @finished_p then
55
+ sprintf("%s %s %s", bytes, transfer_rate, elapsed)
56
+ else
57
+ sprintf("%s %s %s", bytes, transfer_rate, eta)
58
+ end
59
+ end
60
+
61
+ def fmt_title
62
+ @title[0,(@title_width - 1)] + ":"
63
+ end
64
+
65
+ def convert_bytes (bytes)
66
+ if bytes < 1024
67
+ sprintf("%6dB", bytes)
68
+ elsif bytes < 1024 * 1000 # 1000kb
69
+ sprintf("%5.1fKB", bytes.to_f / 1024)
70
+ elsif bytes < 1024 * 1024 * 1000 # 1000mb
71
+ sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
72
+ else
73
+ sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
74
+ end
75
+ end
76
+
77
+ def transfer_rate
78
+ bytes_per_second = @current.to_f / (Time.now - @start_time)
79
+ sprintf("%s/s", convert_bytes(bytes_per_second))
80
+ end
81
+
82
+ def bytes
83
+ convert_bytes(@current)
84
+ end
85
+
86
+ def format_time (t)
87
+ t = t.to_i
88
+ sec = t % 60
89
+ min = (t / 60) % 60
90
+ hour = t / 3600
91
+ sprintf("%02d:%02d:%02d", hour, min, sec);
92
+ end
93
+
94
+ # ETA stands for Estimated Time of Arrival.
95
+ def eta
96
+ if @current == 0
97
+ "ETA: --:--:--"
98
+ else
99
+ elapsed = Time.now - @start_time
100
+ eta = elapsed * @total / @current - elapsed;
101
+ sprintf("ETA: %s", format_time(eta))
102
+ end
103
+ end
104
+
105
+ def elapsed
106
+ elapsed = Time.now - @start_time
107
+ sprintf("Time: %s", format_time(elapsed))
108
+ end
109
+
110
+ def eol
111
+ if @finished_p then "\n" else "\r" end
112
+ end
113
+
114
+ def do_percentage
115
+ if @total.zero?
116
+ 100
117
+ else
118
+ @current * 100 / @total
119
+ end
120
+ end
121
+
122
+ def get_width
123
+ # FIXME: I don't know how portable it is.
124
+ default_width = 80
125
+ begin
126
+ tiocgwinsz = 0x5413
127
+ data = [0, 0, 0, 0].pack("SSSS")
128
+ if @out.ioctl(tiocgwinsz, data) >= 0 then
129
+ rows, cols, xpixels, ypixels = data.unpack("SSSS")
130
+ if cols >= 0 then cols else default_width end
131
+ else
132
+ default_width
133
+ end
134
+ rescue Exception
135
+ default_width
136
+ end
137
+ end
138
+
139
+ def show
140
+ arguments = @format_arguments.map {|method|
141
+ method = sprintf("fmt_%s", method)
142
+ send(method)
143
+ }
144
+ line = sprintf(@format, *arguments)
145
+
146
+ width = get_width
147
+ if line.length == width - 1
148
+ @out.print(line + eol)
149
+ @out.flush
150
+ elsif line.length >= width
151
+ @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
152
+ if @terminal_width == 0 then @out.print(line + eol) else show end
153
+ else # line.length < width - 1
154
+ @terminal_width += width - line.length + 1
155
+ show
156
+ end
157
+ @previous_time = Time.now
158
+ end
159
+
160
+ def show_if_needed
161
+ if @total.zero?
162
+ cur_percentage = 100
163
+ prev_percentage = 0
164
+ else
165
+ cur_percentage = (@current * 100 / @total).to_i
166
+ prev_percentage = (@previous * 100 / @total).to_i
167
+ end
168
+
169
+ # Use "!=" instead of ">" to support negative changes
170
+ if cur_percentage != prev_percentage ||
171
+ Time.now - @previous_time >= 1 || @finished_p
172
+ show
173
+ end
174
+ end
175
+
176
+ public
177
+ def clear
178
+ @out.print "\r"
179
+ @out.print(" " * (get_width - 1))
180
+ @out.print "\r"
181
+ end
182
+
183
+ def finish
184
+ @current = @total
185
+ @finished_p = true
186
+ show
187
+ end
188
+
189
+ def finished?
190
+ @finished_p
191
+ end
192
+
193
+ def file_transfer_mode
194
+ @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
195
+ end
196
+
197
+ def format= (format)
198
+ @format = format
199
+ end
200
+
201
+ def format_arguments= (arguments)
202
+ @format_arguments = arguments
203
+ end
204
+
205
+ def halt
206
+ @finished_p = true
207
+ show
208
+ end
209
+
210
+ def inc (step = 1)
211
+ @current += step
212
+ @current = @total if @current > @total
213
+ show_if_needed
214
+ @previous = @current
215
+ end
216
+
217
+ def set (count)
218
+ count = 0 if count < 0
219
+ count = @total if count > @total
220
+
221
+ @current = count
222
+ show_if_needed
223
+ @previous = @current
224
+ end
225
+
226
+ def inspect
227
+ "#<ProgressBar:#{@current}/#{@total}>"
228
+ end
229
+ end
230
+
231
+ class ReversedProgressBar < ProgressBar
232
+ def do_percentage
233
+ 100 - super
234
+ end
235
+ end
236
+
@@ -0,0 +1,51 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ # The base class of an aggregator. This class provides the interface to which
4
+ # every aggregator should comply (by simply subclassing this class).
5
+ #
6
+ # When building an aggregator, do not forget that RequestLogAnalyzer can run in
7
+ # single line mode or in combined requests mode. Make sure your aggregator can
8
+ # handle both cases, or raise an exception if RLA is rnning in the wrong mode.
9
+ # Calling options[:combined_requests] tells you if RLA is running in combined
10
+ # requests mode, otherwise it is running in single line mode.
11
+ class Base
12
+
13
+ include RequestLogAnalyzer::FileFormat::Awareness
14
+
15
+ attr_reader :options
16
+ attr_reader :source
17
+
18
+ # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
19
+ # It will include the specific file format module.
20
+ def initialize(source, options = {})
21
+ @source = source
22
+ self.register_file_format(source.file_format)
23
+ @options = options
24
+ end
25
+
26
+ # The prepare function is called just before parsing starts. This function
27
+ # can be used to initialie variables, etc.
28
+ def prepare
29
+ end
30
+
31
+ # The aggregate function is called for every request.
32
+ # Implement the aggregating functionality in this method
33
+ def aggregate(request)
34
+ end
35
+
36
+ # The finalize function is called after all sources are parsed and no more
37
+ # requests will be passed to the aggregator
38
+ def finalize
39
+ end
40
+
41
+ # The warning method is called if the parser eits a warning.
42
+ def warning(type, message, lineno)
43
+ end
44
+
45
+ # The report function is called at the end. Implement any result reporting
46
+ # in this function.
47
+ def report(output = STDOUT, report_width = 80, color = false)
48
+ end
49
+
50
+ end
51
+ end
@@ -0,0 +1,97 @@
1
+ require 'rubygems'
2
+ require 'activerecord'
3
+
4
+ module RequestLogAnalyzer::Aggregator
5
+
6
+ class Database < Base
7
+
8
+ attr_reader :request_id
9
+
10
+ def prepare
11
+ ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
12
+
13
+ File.unlink(options[:database]) if File.exist?(options[:database])
14
+ create_database_schema!
15
+
16
+ @request_id = 0
17
+ end
18
+
19
+ def aggregate(request)
20
+ @request_id += 1
21
+
22
+ request.lines.each do |line|
23
+ class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
24
+
25
+ attributes = line.reject { |k, v| [:line_type].include?(k) }
26
+ attributes[:request_id] = @request_id if options[:combined_requests]
27
+ file_format.class.const_get(class_name).create!(attributes)
28
+ end
29
+ rescue SQLite3::SQLException => e
30
+ raise Interrupt, e.message
31
+ end
32
+
33
+ def finalize
34
+ ActiveRecord::Base.remove_connection
35
+ end
36
+
37
+ def warning(type, message, lineno)
38
+ file_format.class::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
39
+ end
40
+
41
+ def report(output = STDOUT, report_width = 80, color = false)
42
+ output << "\n"
43
+ output << green("━" * report_width, color) + "\n"
44
+ output << "A database file has been created with all parsed request information.\n"
45
+ output << "To execute queries on this database, run the following command:\n"
46
+ output << " $ sqlite3 #{options[:database]}\n"
47
+ output << "\n"
48
+ end
49
+
50
+ protected
51
+
52
+ def create_database_table(name, definition)
53
+ ActiveRecord::Migration.verbose = options[:debug]
54
+ ActiveRecord::Migration.create_table("#{name}_lines") do |t|
55
+ t.column(:request_id, :integer) #if options[:combined_requests]
56
+ t.column(:lineno, :integer)
57
+ definition.captures.each do |capture|
58
+ t.column(capture[:name], column_type(capture))
59
+ end
60
+ end
61
+ end
62
+
63
+ def create_warning_table_and_class
64
+ ActiveRecord::Migration.verbose = options[:debug]
65
+ ActiveRecord::Migration.create_table("warnings") do |t|
66
+ t.string :warning_type, :limit => 30, :null => false
67
+ t.string :message
68
+ t.integer :lineno
69
+ end
70
+
71
+ file_format.class.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?('Warning')
72
+ end
73
+
74
+ def create_activerecord_class(name, definition)
75
+ class_name = "#{name}_line".camelize
76
+ file_format.class.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?(class_name)
77
+ end
78
+
79
+ def create_database_schema!
80
+ file_format.line_definitions.each do |name, definition|
81
+ create_database_table(name, definition)
82
+ create_activerecord_class(name, definition)
83
+ end
84
+
85
+ create_warning_table_and_class
86
+ end
87
+
88
+ def column_type(capture)
89
+ case capture[:type]
90
+ when :sec; :double
91
+ when :msec; :double
92
+ when :float; :double
93
+ else capture[:type]
94
+ end
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,25 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ class Echo < Base
4
+
5
+ def prepare
6
+ @warnings = ""
7
+ end
8
+
9
+ def aggregate(request)
10
+ puts "\nRequest: " + request.inspect
11
+ end
12
+
13
+ def warning(type, message, lineno)
14
+ @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
15
+ end
16
+
17
+ def report(output=STDOUT, report_width = 80, color = false)
18
+ output << "\n"
19
+ output << "Warnings during parsing:\n"
20
+ output << green("━" * report_width, color) + "\n"
21
+ output << @warnings + "\n"
22
+ end
23
+
24
+ end
25
+ end
@@ -0,0 +1,116 @@
1
+ require File.dirname(__FILE__) + '/../tracker/base'
2
+
3
+ module RequestLogAnalyzer::Aggregator
4
+
5
+ class Summarizer < Base
6
+
7
+ class Definer
8
+
9
+ attr_reader :trackers
10
+
11
+ def initialize
12
+ @trackers = []
13
+ end
14
+
15
+ def method_missing(tracker_method, *args)
16
+ track(tracker_method, args.first)
17
+ end
18
+
19
+ def category(category_field, options = {})
20
+ if category_field.kind_of?(Symbol)
21
+ track(:category, options.merge(:category => category_field))
22
+ elsif category_field.kind_of?(Hash)
23
+ track(:category, category_field.merge(options))
24
+ end
25
+ end
26
+
27
+ def duration(duration_field, options = {})
28
+ if duration_field.kind_of?(Symbol)
29
+ track(:duration, options.merge(:duration => duration_field))
30
+ elsif duration_field.kind_of?(Hash)
31
+ track(:duration, duration_field.merge(options))
32
+ end
33
+ end
34
+
35
+ def track(tracker_klass, options = {})
36
+ require "#{File.dirname(__FILE__)}/../tracker/#{tracker_klass}"
37
+ tracker_klass = RequestLogAnalyzer::Tracker.const_get(tracker_klass.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) if tracker_klass.kind_of?(Symbol)
38
+ @trackers << tracker_klass.new(options)
39
+ end
40
+ end
41
+
42
+ attr_reader :trackers
43
+ attr_reader :warnings_encountered
44
+
45
+ def initialize(source, options = {})
46
+ super(source, options)
47
+ @warnings_encountered = {}
48
+ @trackers = source.file_format.report_trackers
49
+ setup
50
+ end
51
+
52
+ def setup
53
+ end
54
+
55
+ def prepare
56
+ raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
57
+ @trackers.each { |tracker| tracker.prepare }
58
+ end
59
+
60
+ def aggregate(request)
61
+ @trackers.each do |tracker|
62
+ tracker.update(request) if tracker.should_update?(request)
63
+ end
64
+ end
65
+
66
+ def finalize
67
+ @trackers.each { |tracker| tracker.finalize }
68
+ end
69
+
70
+ def report(output=STDOUT, report_width = 80, color = false)
71
+ report_header(output, report_width, color)
72
+ if source.parsed_requests - source.skipped_requests > 0
73
+ @trackers.each { |tracker| tracker.report(output, report_width, color) }
74
+ else
75
+ output << "\n"
76
+ output << "There were no requests analyzed.\n"
77
+ end
78
+ report_footer(output, report_width, color)
79
+ end
80
+
81
+ def report_header(output=STDOUT, report_width = 80, color = false)
82
+ output << "Request summary\n"
83
+ output << green("━" * report_width, color) + "\n"
84
+ output << "Parsed lines: #{green(source.parsed_lines, color)}\n"
85
+ output << "Parsed requests: #{green(source.parsed_requests, color)}\n" if options[:combined_requests]
86
+ output << "Skipped requests: #{green(source.skipped_requests, color)}\n" if source.skipped_requests > 0
87
+ if has_warnings?
88
+ output << "Warnings: " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
89
+ end
90
+ output << "\n"
91
+ end
92
+
93
+ def report_footer(output=STDOUT, report_width = 80, color = false)
94
+ output << "\n"
95
+ if has_serious_warnings?
96
+ output << green("━" * report_width, color) + "\n"
97
+ output << "Multiple warnings were encountered during parsing. Possibly, your logging " + "\n"
98
+ output << "is not setup correctly. Visit this website for logging configuration tips:" + "\n"
99
+ output << blue("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging", color) + "\n"
100
+ end
101
+ end
102
+
103
+ def has_warnings?
104
+ @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
105
+ end
106
+
107
+ def has_serious_warnings?
108
+ @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 10
109
+ end
110
+
111
+ def warning(type, message, lineno)
112
+ @warnings_encountered[type] ||= 0
113
+ @warnings_encountered[type] += 1
114
+ end
115
+ end
116
+ end