request-log-analyzer 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/DESIGN +14 -0
  2. data/HACKING +7 -0
  3. data/LICENSE +20 -0
  4. data/README.textile +36 -0
  5. data/Rakefile +5 -0
  6. data/bin/request-log-analyzer +123 -0
  7. data/lib/cli/bashcolorizer.rb +60 -0
  8. data/lib/cli/command_line_arguments.rb +301 -0
  9. data/lib/cli/progressbar.rb +236 -0
  10. data/lib/request_log_analyzer.rb +14 -0
  11. data/lib/request_log_analyzer/aggregator/base.rb +45 -0
  12. data/lib/request_log_analyzer/aggregator/database.rb +148 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +25 -0
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +116 -0
  15. data/lib/request_log_analyzer/controller.rb +201 -0
  16. data/lib/request_log_analyzer/file_format.rb +81 -0
  17. data/lib/request_log_analyzer/file_format/merb.rb +33 -0
  18. data/lib/request_log_analyzer/file_format/rails.rb +90 -0
  19. data/lib/request_log_analyzer/filter/base.rb +29 -0
  20. data/lib/request_log_analyzer/filter/field.rb +36 -0
  21. data/lib/request_log_analyzer/filter/timespan.rb +32 -0
  22. data/lib/request_log_analyzer/line_definition.rb +159 -0
  23. data/lib/request_log_analyzer/log_parser.rb +173 -0
  24. data/lib/request_log_analyzer/log_processor.rb +121 -0
  25. data/lib/request_log_analyzer/request.rb +95 -0
  26. data/lib/request_log_analyzer/source/base.rb +42 -0
  27. data/lib/request_log_analyzer/source/log_file.rb +170 -0
  28. data/lib/request_log_analyzer/tracker/base.rb +54 -0
  29. data/lib/request_log_analyzer/tracker/category.rb +71 -0
  30. data/lib/request_log_analyzer/tracker/duration.rb +81 -0
  31. data/lib/request_log_analyzer/tracker/hourly_spread.rb +80 -0
  32. data/lib/request_log_analyzer/tracker/timespan.rb +54 -0
  33. data/spec/controller_spec.rb +40 -0
  34. data/spec/database_inserter_spec.rb +101 -0
  35. data/spec/file_format_spec.rb +78 -0
  36. data/spec/file_formats/spec_format.rb +26 -0
  37. data/spec/filter_spec.rb +137 -0
  38. data/spec/fixtures/merb.log +84 -0
  39. data/spec/fixtures/multiple_files_1.log +5 -0
  40. data/spec/fixtures/multiple_files_2.log +2 -0
  41. data/spec/fixtures/rails_1x.log +59 -0
  42. data/spec/fixtures/rails_22.log +12 -0
  43. data/spec/fixtures/rails_22_cached.log +10 -0
  44. data/spec/fixtures/rails_unordered.log +24 -0
  45. data/spec/fixtures/syslog_1x.log +5 -0
  46. data/spec/fixtures/test_file_format.log +13 -0
  47. data/spec/fixtures/test_language_combined.log +14 -0
  48. data/spec/fixtures/test_order.log +16 -0
  49. data/spec/line_definition_spec.rb +124 -0
  50. data/spec/log_parser_spec.rb +68 -0
  51. data/spec/log_processor_spec.rb +57 -0
  52. data/spec/merb_format_spec.rb +38 -0
  53. data/spec/rails_format_spec.rb +76 -0
  54. data/spec/request_spec.rb +72 -0
  55. data/spec/spec_helper.rb +67 -0
  56. data/spec/summarizer_spec.rb +9 -0
  57. data/tasks/github-gem.rake +177 -0
  58. data/tasks/request_log_analyzer.rake +10 -0
  59. data/tasks/rspec.rake +6 -0
  60. metadata +135 -0
@@ -0,0 +1,236 @@
1
+ #
2
+ # Ruby/ProgressBar - a text progress bar library
3
+ #
4
+ # Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
5
+ # All rights reserved.
6
+ # This is free software with ABSOLUTELY NO WARRANTY.
7
+ #
8
+ # You can redistribute it and/or modify it under the terms
9
+ # of Ruby's license.
10
+ #
11
+
12
+ class ProgressBar
13
+ VERSION = "0.9"
14
+
15
+ def initialize (title, total, out = STDERR)
16
+ @title = title
17
+ @total = total
18
+ @out = out
19
+ @terminal_width = 80
20
+ @bar_mark = '='
21
+ @current = 0
22
+ @previous = 0
23
+ @finished_p = false
24
+ @start_time = Time.now
25
+ @previous_time = @start_time
26
+ @title_width = 24
27
+ @format = "%-#{@title_width}s %3d%% %s %s"
28
+ @format_arguments = [:title, :percentage, :bar, :stat]
29
+ clear
30
+ show
31
+ end
32
+ attr_reader :title
33
+ attr_reader :current
34
+ attr_reader :total
35
+ attr_accessor :start_time
36
+
37
+ private
38
+ def fmt_bar
39
+ bar_width = do_percentage * @terminal_width / 100
40
+ sprintf("[%s%s]",
41
+ @bar_mark * bar_width,
42
+ " " * (@terminal_width - bar_width))
43
+ end
44
+
45
+ def fmt_percentage
46
+ do_percentage
47
+ end
48
+
49
+ def fmt_stat
50
+ if @finished_p then elapsed else eta end
51
+ end
52
+
53
+ def fmt_stat_for_file_transfer
54
+ if @finished_p then
55
+ sprintf("%s %s %s", bytes, transfer_rate, elapsed)
56
+ else
57
+ sprintf("%s %s %s", bytes, transfer_rate, eta)
58
+ end
59
+ end
60
+
61
+ def fmt_title
62
+ @title[0,(@title_width - 1)] + ":"
63
+ end
64
+
65
+ def convert_bytes (bytes)
66
+ if bytes < 1024
67
+ sprintf("%6dB", bytes)
68
+ elsif bytes < 1024 * 1000 # 1000kb
69
+ sprintf("%5.1fKB", bytes.to_f / 1024)
70
+ elsif bytes < 1024 * 1024 * 1000 # 1000mb
71
+ sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
72
+ else
73
+ sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
74
+ end
75
+ end
76
+
77
+ def transfer_rate
78
+ bytes_per_second = @current.to_f / (Time.now - @start_time)
79
+ sprintf("%s/s", convert_bytes(bytes_per_second))
80
+ end
81
+
82
+ def bytes
83
+ convert_bytes(@current)
84
+ end
85
+
86
+ def format_time (t)
87
+ t = t.to_i
88
+ sec = t % 60
89
+ min = (t / 60) % 60
90
+ hour = t / 3600
91
+ sprintf("%02d:%02d:%02d", hour, min, sec);
92
+ end
93
+
94
+ # ETA stands for Estimated Time of Arrival.
95
+ def eta
96
+ if @current == 0
97
+ "ETA: --:--:--"
98
+ else
99
+ elapsed = Time.now - @start_time
100
+ eta = elapsed * @total / @current - elapsed;
101
+ sprintf("ETA: %s", format_time(eta))
102
+ end
103
+ end
104
+
105
+ def elapsed
106
+ elapsed = Time.now - @start_time
107
+ sprintf("Time: %s", format_time(elapsed))
108
+ end
109
+
110
+ def eol
111
+ if @finished_p then "\n" else "\r" end
112
+ end
113
+
114
+ def do_percentage
115
+ if @total.zero?
116
+ 100
117
+ else
118
+ @current * 100 / @total
119
+ end
120
+ end
121
+
122
+ def get_width
123
+ # FIXME: I don't know how portable it is.
124
+ default_width = 80
125
+ begin
126
+ tiocgwinsz = 0x5413
127
+ data = [0, 0, 0, 0].pack("SSSS")
128
+ if @out.ioctl(tiocgwinsz, data) >= 0 then
129
+ rows, cols, xpixels, ypixels = data.unpack("SSSS")
130
+ if cols >= 0 then cols else default_width end
131
+ else
132
+ default_width
133
+ end
134
+ rescue Exception
135
+ default_width
136
+ end
137
+ end
138
+
139
+ def show
140
+ arguments = @format_arguments.map {|method|
141
+ method = sprintf("fmt_%s", method)
142
+ send(method)
143
+ }
144
+ line = sprintf(@format, *arguments)
145
+
146
+ width = get_width
147
+ if line.length == width - 1
148
+ @out.print(line + eol)
149
+ @out.flush
150
+ elsif line.length >= width
151
+ @terminal_width = [@terminal_width - (line.length - width + 1), 0].max
152
+ if @terminal_width == 0 then @out.print(line + eol) else show end
153
+ else # line.length < width - 1
154
+ @terminal_width += width - line.length + 1
155
+ show
156
+ end
157
+ @previous_time = Time.now
158
+ end
159
+
160
+ def show_if_needed
161
+ if @total.zero?
162
+ cur_percentage = 100
163
+ prev_percentage = 0
164
+ else
165
+ cur_percentage = (@current * 100 / @total).to_i
166
+ prev_percentage = (@previous * 100 / @total).to_i
167
+ end
168
+
169
+ # Use "!=" instead of ">" to support negative changes
170
+ if cur_percentage != prev_percentage ||
171
+ Time.now - @previous_time >= 1 || @finished_p
172
+ show
173
+ end
174
+ end
175
+
176
+ public
177
+ def clear
178
+ @out.print "\r"
179
+ @out.print(" " * (get_width - 1))
180
+ @out.print "\r"
181
+ end
182
+
183
+ def finish
184
+ @current = @total
185
+ @finished_p = true
186
+ show
187
+ end
188
+
189
+ def finished?
190
+ @finished_p
191
+ end
192
+
193
+ def file_transfer_mode
194
+ @format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
195
+ end
196
+
197
+ def format= (format)
198
+ @format = format
199
+ end
200
+
201
+ def format_arguments= (arguments)
202
+ @format_arguments = arguments
203
+ end
204
+
205
+ def halt
206
+ @finished_p = true
207
+ show
208
+ end
209
+
210
+ def inc (step = 1)
211
+ @current += step
212
+ @current = @total if @current > @total
213
+ show_if_needed
214
+ @previous = @current
215
+ end
216
+
217
+ def set (count)
218
+ count = 0 if count < 0
219
+ count = @total if count > @total
220
+
221
+ @current = count
222
+ show_if_needed
223
+ @previous = @current
224
+ end
225
+
226
+ def inspect
227
+ "#<ProgressBar:#{@current}/#{@total}>"
228
+ end
229
+ end
230
+
231
+ class ReversedProgressBar < ProgressBar
232
+ def do_percentage
233
+ 100 - super
234
+ end
235
+ end
236
+
@@ -0,0 +1,14 @@
1
+ require 'date'
2
+ require File.dirname(__FILE__) + '/cli/progressbar'
3
+ require File.dirname(__FILE__) + '/cli/bashcolorizer'
4
+
5
+ require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
6
+ require File.dirname(__FILE__) + '/request_log_analyzer/line_definition'
7
+ require File.dirname(__FILE__) + '/request_log_analyzer/request'
8
+ require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
9
+ require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
10
+ require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/summarizer'
11
+ require File.dirname(__FILE__) + '/request_log_analyzer/filter/base'
12
+ require File.dirname(__FILE__) + '/request_log_analyzer/controller'
13
+ require File.dirname(__FILE__) + '/request_log_analyzer/source/base'
14
+ require File.dirname(__FILE__) + '/request_log_analyzer/source/log_file'
@@ -0,0 +1,45 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ # The base class of an aggregator. This class provides the interface to which
4
+ # every aggregator should comply (by simply subclassing this class).
5
+ class Base
6
+
7
+ include RequestLogAnalyzer::FileFormat::Awareness
8
+
9
+ attr_reader :options
10
+ attr_reader :source
11
+
12
+ # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
13
+ # It will include the specific file format module.
14
+ def initialize(source, options = {})
15
+ @source = source
16
+ self.register_file_format(source.file_format)
17
+ @options = options
18
+ end
19
+
20
+ # The prepare function is called just before parsing starts. This function
21
+ # can be used to initialie variables, etc.
22
+ def prepare
23
+ end
24
+
25
+ # The aggregate function is called for every request.
26
+ # Implement the aggregating functionality in this method
27
+ def aggregate(request)
28
+ end
29
+
30
+ # The finalize function is called after all sources are parsed and no more
31
+ # requests will be passed to the aggregator
32
+ def finalize
33
+ end
34
+
35
+ # The warning method is called if the parser eits a warning.
36
+ def warning(type, message, lineno)
37
+ end
38
+
39
+ # The report function is called at the end. Implement any result reporting
40
+ # in this function.
41
+ def report(output = STDOUT, report_width = 80, color = false)
42
+ end
43
+
44
+ end
45
+ end
@@ -0,0 +1,148 @@
1
+ require 'rubygems'
2
+ require 'activerecord'
3
+
4
+ module RequestLogAnalyzer::Aggregator
5
+
6
+ # The database aggregator will create an SQLite3 database with all parsed request information.
7
+ #
8
+ # The prepare method will create a database schema according to the file format definitions.
9
+ # It will also create ActiveRecord::Base subclasses to interact with the created tables.
10
+ # Then, the aggregate method will be called for every parsed request. The information of
11
+ # these requests is inserted into the tables using the ActiveRecord classes.
12
+ #
13
+ # A requests table will be created, in which a record is inserted for every parsed request.
14
+ # For every line type, a separate table will be created with a request_id field to point to
15
+ # the request record, and a field for every parsed value. Finally, a warnings table will be
16
+ # created to log all parse warnings.
17
+ class Database < Base
18
+
19
+ # Establishes a connection to the database and creates the necessary database schema for the
20
+ # current file format
21
+ def prepare
22
+ ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
23
+ File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
24
+ create_database_schema!
25
+ end
26
+
27
+ # Aggregates a request into the database
28
+ # This will create a record in the requests table and create a record for every line that has been parsed,
29
+ # in which the captured values will be stored.
30
+ def aggregate(request)
31
+ @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
32
+ request.lines.each do |line|
33
+ attributes = line.reject { |k, v| [:line_type].include?(k) }
34
+ @request_object.send("#{line[:line_type]}_lines").build(attributes)
35
+ end
36
+ @request_object.save!
37
+ rescue SQLite3::SQLException => e
38
+ raise Interrupt, e.message
39
+ end
40
+
41
+ # Finalizes the aggregator by closing the connection to the database
42
+ def finalize
43
+ @request_count = @orm_module::Request.count
44
+ ActiveRecord::Base.remove_connection
45
+ end
46
+
47
+ # Records w warining in the warnings table.
48
+ def warning(type, message, lineno)
49
+ @orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
50
+ end
51
+
52
+ # Prints a short report of what has been inserted into the database
53
+ def report(output = STDOUT, report_width = 80, color = false)
54
+ output << "\n"
55
+ output << green("━" * report_width, color) + "\n"
56
+ output << "A database file has been created with all parsed request information.\n"
57
+ output << "#{@request_count} requests have been added to the database.\n"
58
+ output << "To execute queries on this database, run the following command:\n"
59
+ output << " $ sqlite3 #{options[:database]}\n"
60
+ output << "\n"
61
+ end
62
+
63
+ protected
64
+
65
+ # This function creates a database table for a given line definition.
66
+ # It will create a field for every capture in the line, and adds a lineno field to indicate at
67
+ # what line in the original file the line was found, and a request_id to link lines related
68
+ # to the same request. It will also create an index in the request_id field to speed up queries.
69
+ def create_database_table(name, definition)
70
+ ActiveRecord::Migration.verbose = options[:debug]
71
+ ActiveRecord::Migration.create_table("#{name}_lines") do |t|
72
+ t.column(:request_id, :integer)
73
+ t.column(:lineno, :integer)
74
+ definition.captures.each do |capture|
75
+ t.column(capture[:name], column_type(capture))
76
+ end
77
+ end
78
+ ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
79
+ end
80
+
81
+ # Creates an ActiveRecord class for a given line definition.
82
+ # A subclass of ActiveRecord::Base is created and an association with the Request class is
83
+ # created using belongs_to / has_many. This association will later be used to create records
84
+ # in the corresponding table. This table should already be created before this method is called.
85
+ def create_activerecord_class(name, definition)
86
+ class_name = "#{name}_line".camelize
87
+ klass = Class.new(ActiveRecord::Base)
88
+ klass.send(:belongs_to, :request)
89
+ @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
90
+ @request_class.send(:has_many, "#{name}_lines".to_sym)
91
+ end
92
+
93
+ # Creates a requests table, in which a record is created for every request. It also creates an
94
+ # ActiveRecord::Base class to communicate with this table.
95
+ def create_request_table_and_class
96
+ ActiveRecord::Migration.verbose = options[:debug]
97
+ ActiveRecord::Migration.create_table("requests") do |t|
98
+ t.integer :first_lineno
99
+ t.integer :last_lineno
100
+ end
101
+
102
+ @orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')
103
+ @request_class = @orm_module.const_get('Request')
104
+ end
105
+
106
+ # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
107
+ def create_warning_table_and_class
108
+ ActiveRecord::Migration.verbose = options[:debug]
109
+ ActiveRecord::Migration.create_table("warnings") do |t|
110
+ t.string :warning_type, :limit => 30, :null => false
111
+ t.string :message
112
+ t.integer :lineno
113
+ end
114
+
115
+ @orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
116
+ end
117
+
118
+ # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
119
+ # file format definition. These ORM classes will later be used to create records in the database.
120
+ def create_database_schema!
121
+
122
+ if file_format.class.const_defined?('Database')
123
+ @orm_module = file_format.class.const_get('Database')
124
+ else
125
+ @orm_module = file_format.class.const_set('Database', Module.new)
126
+ end
127
+
128
+ create_request_table_and_class
129
+ create_warning_table_and_class
130
+
131
+ file_format.line_definitions.each do |name, definition|
132
+ create_database_table(name, definition)
133
+ create_activerecord_class(name, definition)
134
+ end
135
+ end
136
+
137
+ # Function to determine the column type for a field
138
+ # TODO: make more robust / include in file-format definition
139
+ def column_type(capture)
140
+ case capture[:type]
141
+ when :sec; :double
142
+ when :msec; :double
143
+ when :float; :double
144
+ else capture[:type]
145
+ end
146
+ end
147
+ end
148
+ end
@@ -0,0 +1,25 @@
1
+ module RequestLogAnalyzer::Aggregator
2
+
3
+ class Echo < Base
4
+
5
+ def prepare
6
+ @warnings = ""
7
+ end
8
+
9
+ def aggregate(request)
10
+ puts "\nRequest: " + request.inspect
11
+ end
12
+
13
+ def warning(type, message, lineno)
14
+ @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}\n"
15
+ end
16
+
17
+ def report(output=STDOUT, report_width = 80, color = false)
18
+ output << "\n"
19
+ output << "Warnings during parsing:\n"
20
+ output << green("━" * report_width, color) + "\n"
21
+ output << @warnings + "\n"
22
+ end
23
+
24
+ end
25
+ end