ngmoco-request-log-analyzer 1.4.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +10 -0
- data/DESIGN.rdoc +41 -0
- data/LICENSE +20 -0
- data/README.rdoc +39 -0
- data/Rakefile +8 -0
- data/bin/request-log-analyzer +114 -0
- data/lib/cli/command_line_arguments.rb +301 -0
- data/lib/cli/database_console.rb +26 -0
- data/lib/cli/database_console_init.rb +43 -0
- data/lib/cli/progressbar.rb +213 -0
- data/lib/cli/tools.rb +46 -0
- data/lib/request_log_analyzer.rb +44 -0
- data/lib/request_log_analyzer/aggregator.rb +49 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
- data/lib/request_log_analyzer/aggregator/echo.rb +29 -0
- data/lib/request_log_analyzer/aggregator/summarizer.rb +175 -0
- data/lib/request_log_analyzer/controller.rb +332 -0
- data/lib/request_log_analyzer/database.rb +102 -0
- data/lib/request_log_analyzer/database/base.rb +115 -0
- data/lib/request_log_analyzer/database/connection.rb +38 -0
- data/lib/request_log_analyzer/database/request.rb +22 -0
- data/lib/request_log_analyzer/database/source.rb +13 -0
- data/lib/request_log_analyzer/database/warning.rb +14 -0
- data/lib/request_log_analyzer/file_format.rb +160 -0
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +71 -0
- data/lib/request_log_analyzer/file_format/apache.rb +141 -0
- data/lib/request_log_analyzer/file_format/merb.rb +67 -0
- data/lib/request_log_analyzer/file_format/rack.rb +11 -0
- data/lib/request_log_analyzer/file_format/rails.rb +176 -0
- data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
- data/lib/request_log_analyzer/filter.rb +30 -0
- data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
- data/lib/request_log_analyzer/filter/field.rb +42 -0
- data/lib/request_log_analyzer/filter/timespan.rb +45 -0
- data/lib/request_log_analyzer/line_definition.rb +111 -0
- data/lib/request_log_analyzer/log_processor.rb +99 -0
- data/lib/request_log_analyzer/mailer.rb +62 -0
- data/lib/request_log_analyzer/output.rb +113 -0
- data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
- data/lib/request_log_analyzer/output/html.rb +184 -0
- data/lib/request_log_analyzer/request.rb +175 -0
- data/lib/request_log_analyzer/source.rb +72 -0
- data/lib/request_log_analyzer/source/database_loader.rb +87 -0
- data/lib/request_log_analyzer/source/log_parser.rb +274 -0
- data/lib/request_log_analyzer/tracker.rb +206 -0
- data/lib/request_log_analyzer/tracker/duration.rb +104 -0
- data/lib/request_log_analyzer/tracker/frequency.rb +95 -0
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +107 -0
- data/lib/request_log_analyzer/tracker/timespan.rb +81 -0
- data/lib/request_log_analyzer/tracker/traffic.rb +106 -0
- data/request-log-analyzer.gemspec +40 -0
- data/spec/database.yml +23 -0
- data/spec/fixtures/apache_combined.log +5 -0
- data/spec/fixtures/apache_common.log +10 -0
- data/spec/fixtures/decompression.log +12 -0
- data/spec/fixtures/decompression.log.bz2 +0 -0
- data/spec/fixtures/decompression.log.gz +0 -0
- data/spec/fixtures/decompression.log.zip +0 -0
- data/spec/fixtures/decompression.tar.gz +0 -0
- data/spec/fixtures/decompression.tgz +0 -0
- data/spec/fixtures/header_and_footer.log +6 -0
- data/spec/fixtures/merb.log +84 -0
- data/spec/fixtures/merb_prefixed.log +9 -0
- data/spec/fixtures/multiple_files_1.log +5 -0
- data/spec/fixtures/multiple_files_2.log +2 -0
- data/spec/fixtures/rails.db +0 -0
- data/spec/fixtures/rails_1x.log +59 -0
- data/spec/fixtures/rails_22.log +12 -0
- data/spec/fixtures/rails_22_cached.log +10 -0
- data/spec/fixtures/rails_unordered.log +24 -0
- data/spec/fixtures/syslog_1x.log +5 -0
- data/spec/fixtures/test_file_format.log +13 -0
- data/spec/fixtures/test_language_combined.log +14 -0
- data/spec/fixtures/test_order.log +16 -0
- data/spec/integration/command_line_usage_spec.rb +84 -0
- data/spec/integration/munin_plugins_rails_spec.rb +58 -0
- data/spec/integration/scout_spec.rb +151 -0
- data/spec/lib/helpers.rb +52 -0
- data/spec/lib/macros.rb +18 -0
- data/spec/lib/matchers.rb +77 -0
- data/spec/lib/mocks.rb +76 -0
- data/spec/lib/testing_format.rb +46 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
- data/spec/unit/aggregator/summarizer_spec.rb +26 -0
- data/spec/unit/controller/controller_spec.rb +41 -0
- data/spec/unit/controller/log_processor_spec.rb +18 -0
- data/spec/unit/database/base_class_spec.rb +183 -0
- data/spec/unit/database/connection_spec.rb +34 -0
- data/spec/unit/database/database_spec.rb +133 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +49 -0
- data/spec/unit/file_format/apache_format_spec.rb +203 -0
- data/spec/unit/file_format/file_format_api_spec.rb +69 -0
- data/spec/unit/file_format/line_definition_spec.rb +75 -0
- data/spec/unit/file_format/merb_format_spec.rb +52 -0
- data/spec/unit/file_format/rails_format_spec.rb +164 -0
- data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
- data/spec/unit/filter/field_filter_spec.rb +66 -0
- data/spec/unit/filter/filter_spec.rb +17 -0
- data/spec/unit/filter/timespan_filter_spec.rb +58 -0
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/request_spec.rb +111 -0
- data/spec/unit/source/log_parser_spec.rb +119 -0
- data/spec/unit/tracker/duration_tracker_spec.rb +130 -0
- data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
- data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
- data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
- data/spec/unit/tracker/tracker_api_spec.rb +124 -0
- data/spec/unit/tracker/traffic_tracker_spec.rb +107 -0
- data/tasks/github-gem.rake +323 -0
- data/tasks/request_log_analyzer.rake +26 -0
- metadata +220 -0
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
class DatabaseConsole
|
3
|
+
|
4
|
+
IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
|
5
|
+
|
6
|
+
def initialize(arguments)
|
7
|
+
@arguments = arguments
|
8
|
+
end
|
9
|
+
|
10
|
+
def run!
|
11
|
+
libraries = ['irb/completion', 'rubygems', './lib/request_log_analyzer', './lib/cli/database_console_init']
|
12
|
+
libaries_string = libraries.map { |l| "-r #{l}" }.join(' ')
|
13
|
+
|
14
|
+
ENV['RLA_DBCONSOLE_DATABASE'] = @arguments[:database]
|
15
|
+
if @arguments[:apache_format]
|
16
|
+
ENV['RLA_DBCONSOLE_FORMAT'] = 'apache'
|
17
|
+
ENV['RLA_DBCONSOLE_FORMAT_ARGUMENT'] = @arguments[:apache_format]
|
18
|
+
else
|
19
|
+
ENV['RLA_DBCONSOLE_FORMAT'] = @arguments[:format]
|
20
|
+
end
|
21
|
+
# ENV['RLA_DBCONSOLE_FORMAT_ARGS'] = arguments['database']
|
22
|
+
|
23
|
+
exec("#{IRB} #{libaries_string} --simple-prompt")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# Setup the include path
|
2
|
+
$:.unshift(File.dirname(__FILE__) + '/..')
|
3
|
+
|
4
|
+
$database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE'])
|
5
|
+
$database.load_database_schema!
|
6
|
+
$database.register_default_orm_classes!
|
7
|
+
|
8
|
+
require 'cli/tools'
|
9
|
+
|
10
|
+
def wordwrap(string, max = 80, indent = "")
|
11
|
+
strings = [""]
|
12
|
+
string.split(", ").each do |item|
|
13
|
+
if strings.last.length == 0 || strings.last.length + item.length <= max
|
14
|
+
strings.last << item << ', '
|
15
|
+
else
|
16
|
+
strings << (item + ', ')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
strings.map(&:strip).join("\n#{indent}").slice(0..-2)
|
20
|
+
end
|
21
|
+
|
22
|
+
class Request
|
23
|
+
def inspect
|
24
|
+
request_inspect = "Request[id: #{id}]"
|
25
|
+
request_inspect << " <#{lines.first.source.filename}>" if lines.first.source
|
26
|
+
|
27
|
+
inspected_lines = lines.map do |line|
|
28
|
+
inspect_line = " - #{line.line_type} (line #{line.lineno})"
|
29
|
+
if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
|
30
|
+
inspect_attributes = inspect_attributes.map { |(k,v)| "#{k} = #{v.inspect}" }.join(', ')
|
31
|
+
inspect_line << "\n " + wordwrap(inspect_attributes, terminal_width - 6, " ")
|
32
|
+
end
|
33
|
+
inspect_line
|
34
|
+
end
|
35
|
+
|
36
|
+
request_inspect << "\n" << inspected_lines.join("\n") << "\n\n"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
puts "request-log-analyzer database console"
|
41
|
+
puts "-------------------------------------"
|
42
|
+
puts "The following ActiveRecord classes are available:"
|
43
|
+
puts $database.orm_classes.map { |k| k.name.split('::').last }.join(", ")
|
@@ -0,0 +1,213 @@
|
|
1
|
+
#
|
2
|
+
# Ruby/ProgressBar - a text progress bar library
|
3
|
+
#
|
4
|
+
# Copyright (C) 2001-2005 Satoru Takabayashi <satoru@namazu.org>
|
5
|
+
# All rights reserved.
|
6
|
+
# This is free software with ABSOLUTELY NO WARRANTY.
|
7
|
+
#
|
8
|
+
# You can redistribute it and/or modify it under the terms
|
9
|
+
# of Ruby's license.
|
10
|
+
|
11
|
+
module CommandLine
|
12
|
+
class ProgressBar
|
13
|
+
VERSION = "0.9"
|
14
|
+
|
15
|
+
def initialize (title, total, out = STDERR)
|
16
|
+
@title = title
|
17
|
+
@total = total
|
18
|
+
@out = out
|
19
|
+
@terminal_width = 80
|
20
|
+
@bar_mark = '='
|
21
|
+
@current = 0
|
22
|
+
@previous = 0
|
23
|
+
@finished_p = false
|
24
|
+
@start_time = Time.now
|
25
|
+
@previous_time = @start_time
|
26
|
+
@title_width = 24
|
27
|
+
@format = "%-#{@title_width}s %3d%% %s %s"
|
28
|
+
@format_arguments = [:title, :percentage, :bar, :stat]
|
29
|
+
clear
|
30
|
+
show
|
31
|
+
end
|
32
|
+
attr_reader :title
|
33
|
+
attr_reader :current
|
34
|
+
attr_reader :total
|
35
|
+
attr_accessor :start_time
|
36
|
+
|
37
|
+
private
|
38
|
+
def fmt_bar
|
39
|
+
bar_width = do_percentage * @terminal_width / 100
|
40
|
+
sprintf("[%s%s]",
|
41
|
+
@bar_mark * bar_width,
|
42
|
+
" " * (@terminal_width - bar_width))
|
43
|
+
end
|
44
|
+
|
45
|
+
def fmt_percentage
|
46
|
+
do_percentage
|
47
|
+
end
|
48
|
+
|
49
|
+
def fmt_stat
|
50
|
+
if @finished_p then elapsed else eta end
|
51
|
+
end
|
52
|
+
|
53
|
+
def fmt_stat_for_file_transfer
|
54
|
+
if @finished_p then
|
55
|
+
sprintf("%s %s %s", bytes, transfer_rate, elapsed)
|
56
|
+
else
|
57
|
+
sprintf("%s %s %s", bytes, transfer_rate, eta)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def fmt_title
|
62
|
+
@title[0,(@title_width - 1)] + ":"
|
63
|
+
end
|
64
|
+
|
65
|
+
def convert_bytes (bytes)
|
66
|
+
if bytes < 1024
|
67
|
+
sprintf("%6dB", bytes)
|
68
|
+
elsif bytes < 1024 * 1000 # 1000kb
|
69
|
+
sprintf("%5.1fKB", bytes.to_f / 1024)
|
70
|
+
elsif bytes < 1024 * 1024 * 1000 # 1000mb
|
71
|
+
sprintf("%5.1fMB", bytes.to_f / 1024 / 1024)
|
72
|
+
else
|
73
|
+
sprintf("%5.1fGB", bytes.to_f / 1024 / 1024 / 1024)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def transfer_rate
|
78
|
+
bytes_per_second = @current.to_f / (Time.now - @start_time)
|
79
|
+
sprintf("%s/s", convert_bytes(bytes_per_second))
|
80
|
+
end
|
81
|
+
|
82
|
+
def bytes
|
83
|
+
convert_bytes(@current)
|
84
|
+
end
|
85
|
+
|
86
|
+
def format_time (t)
|
87
|
+
t = t.to_i
|
88
|
+
sec = t % 60
|
89
|
+
min = (t / 60) % 60
|
90
|
+
hour = t / 3600
|
91
|
+
sprintf("%02d:%02d:%02d", hour, min, sec);
|
92
|
+
end
|
93
|
+
|
94
|
+
# ETA stands for Estimated Time of Arrival.
|
95
|
+
def eta
|
96
|
+
if @current == 0
|
97
|
+
"ETA: --:--:--"
|
98
|
+
else
|
99
|
+
elapsed = Time.now - @start_time
|
100
|
+
eta = elapsed * @total / @current - elapsed;
|
101
|
+
sprintf("ETA: %s", format_time(eta))
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def elapsed
|
106
|
+
elapsed = Time.now - @start_time
|
107
|
+
sprintf("Time: %s", format_time(elapsed))
|
108
|
+
end
|
109
|
+
|
110
|
+
def eol
|
111
|
+
if @finished_p then "\n" else "\r" end
|
112
|
+
end
|
113
|
+
|
114
|
+
def do_percentage
|
115
|
+
if @total.zero?
|
116
|
+
100
|
117
|
+
else
|
118
|
+
@current * 100 / @total
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
def show
|
123
|
+
arguments = @format_arguments.map {|method|
|
124
|
+
method = sprintf("fmt_%s", method)
|
125
|
+
send(method)
|
126
|
+
}
|
127
|
+
line = sprintf(@format, *arguments)
|
128
|
+
|
129
|
+
width = terminal_width(80)
|
130
|
+
if line.length == width - 1
|
131
|
+
@out.print(line + eol)
|
132
|
+
@out.flush
|
133
|
+
elsif line.length >= width
|
134
|
+
@terminal_width = [@terminal_width - (line.length - width + 1), 0].max
|
135
|
+
if @terminal_width == 0 then @out.print(line + eol) else show end
|
136
|
+
else # line.length < width - 1
|
137
|
+
@terminal_width += width - line.length + 1
|
138
|
+
show
|
139
|
+
end
|
140
|
+
@previous_time = Time.now
|
141
|
+
end
|
142
|
+
|
143
|
+
def show_if_needed
|
144
|
+
if @total.zero?
|
145
|
+
cur_percentage = 100
|
146
|
+
prev_percentage = 0
|
147
|
+
else
|
148
|
+
cur_percentage = (@current * 100 / @total).to_i
|
149
|
+
prev_percentage = (@previous * 100 / @total).to_i
|
150
|
+
end
|
151
|
+
|
152
|
+
# Use "!=" instead of ">" to support negative changes
|
153
|
+
if cur_percentage != prev_percentage ||
|
154
|
+
Time.now - @previous_time >= 1 || @finished_p
|
155
|
+
show
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
public
|
160
|
+
def clear
|
161
|
+
@out.print "\r"
|
162
|
+
@out.print(" " * (terminal_width(80) - 1))
|
163
|
+
@out.print "\r"
|
164
|
+
end
|
165
|
+
|
166
|
+
def finish
|
167
|
+
@current = @total
|
168
|
+
@finished_p = true
|
169
|
+
show
|
170
|
+
end
|
171
|
+
|
172
|
+
def finished?
|
173
|
+
@finished_p
|
174
|
+
end
|
175
|
+
|
176
|
+
def file_transfer_mode
|
177
|
+
@format_arguments = [:title, :percentage, :bar, :stat_for_file_transfer]
|
178
|
+
end
|
179
|
+
|
180
|
+
def format= (format)
|
181
|
+
@format = format
|
182
|
+
end
|
183
|
+
|
184
|
+
def format_arguments= (arguments)
|
185
|
+
@format_arguments = arguments
|
186
|
+
end
|
187
|
+
|
188
|
+
def halt
|
189
|
+
@finished_p = true
|
190
|
+
show
|
191
|
+
end
|
192
|
+
|
193
|
+
def inc (step = 1)
|
194
|
+
@current += step
|
195
|
+
@current = @total if @current > @total
|
196
|
+
show_if_needed
|
197
|
+
@previous = @current
|
198
|
+
end
|
199
|
+
|
200
|
+
def set (count)
|
201
|
+
count = 0 if count < 0
|
202
|
+
count = @total if count > @total
|
203
|
+
|
204
|
+
@current = count
|
205
|
+
show_if_needed
|
206
|
+
@previous = @current
|
207
|
+
end
|
208
|
+
|
209
|
+
def inspect
|
210
|
+
"#<ProgressBar:#{@current}/#{@total}>"
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
data/lib/cli/tools.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# Try to determine the terminal with.
|
2
|
+
# If it is not possible to to so, it returns the default_width.
|
3
|
+
# <tt>default_width</tt> Defaults to 81
|
4
|
+
def terminal_width(default_width = 81)
|
5
|
+
tiocgwinsz = 0x5413
|
6
|
+
data = [0, 0, 0, 0].pack("SSSS")
|
7
|
+
if @out.ioctl(tiocgwinsz, data) >= 0
|
8
|
+
rows, cols, xpixels, ypixels = data.unpack("SSSS")
|
9
|
+
raise unless cols > 0
|
10
|
+
cols
|
11
|
+
else
|
12
|
+
raise
|
13
|
+
end
|
14
|
+
rescue
|
15
|
+
begin
|
16
|
+
IO.popen('stty -a 2>&1') do |pipe|
|
17
|
+
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
18
|
+
raise unless column_line
|
19
|
+
$1.to_i
|
20
|
+
end
|
21
|
+
rescue
|
22
|
+
default_width
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
|
27
|
+
# environment integration.
|
28
|
+
# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
|
29
|
+
# Raises if it cannot find the project folder or if the install_type is now known.
|
30
|
+
def install_rake_tasks(install_type = :rails)
|
31
|
+
if install_type.to_sym == :rails
|
32
|
+
require 'ftools'
|
33
|
+
if File.directory?('./lib/tasks/')
|
34
|
+
File.copy(File.dirname(__FILE__) + '/../../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
|
35
|
+
puts "Installed rake tasks."
|
36
|
+
puts "To use, run: rake rla:report"
|
37
|
+
else
|
38
|
+
puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
|
39
|
+
puts "Installation aborted."
|
40
|
+
end
|
41
|
+
else
|
42
|
+
raise "Cannot perform this install type! (#{install_type.to_s})"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'date'
|
2
|
+
|
3
|
+
# Satisfy ruby 1.9 sensitivity about encoding.
|
4
|
+
Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_to? 'default_external='
|
5
|
+
|
6
|
+
# RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
|
7
|
+
#
|
8
|
+
# - This module itselfs contains some functions to help with class and source file loading.
|
9
|
+
# - The actual application resides in the RequestLogAnalyzer::Controller class.
|
10
|
+
module RequestLogAnalyzer
|
11
|
+
|
12
|
+
# The current version of request-log-analyzer.
|
13
|
+
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
14
|
+
VERSION = "1.4.1"
|
15
|
+
|
16
|
+
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
17
|
+
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
18
|
+
def self.const_missing(const)
|
19
|
+
load_default_class_file(RequestLogAnalyzer, const)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Loads constants that reside in the RequestLogAnalyzer tree using the constant name
|
23
|
+
# and its base constant to determine the filename.
|
24
|
+
# <tt>base</tt>:: The base constant to load the constant from. This should be Foo when the constant Foo::Bar is being loaded.
|
25
|
+
# <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
|
26
|
+
def self.load_default_class_file(base, const)
|
27
|
+
require "#{to_underscore("#{base.name}::#{const}")}"
|
28
|
+
base.const_get(const) if base.const_defined?(const)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Convert a string/symbol in camelcase (RequestLogAnalyzer::Controller) to underscores (request_log_analyzer/controller)
|
32
|
+
# This function can be used to load the file (using require) in which the given constant is defined.
|
33
|
+
# <tt>str</tt>:: The string to convert in the following format: <tt>ModuleName::ClassName</tt>
|
34
|
+
def self.to_underscore(str)
|
35
|
+
str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
36
|
+
end
|
37
|
+
|
38
|
+
# Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
|
39
|
+
# (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
|
40
|
+
# <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
|
41
|
+
def self.to_camelcase(str)
|
42
|
+
str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module RequestLogAnalyzer::Aggregator
|
2
|
+
|
3
|
+
def self.const_missing(const)
|
4
|
+
RequestLogAnalyzer::load_default_class_file(self, const)
|
5
|
+
end
|
6
|
+
|
7
|
+
# The base class of an aggregator. This class provides the interface to which
|
8
|
+
# every aggregator should comply (by simply subclassing this class).
|
9
|
+
class Base
|
10
|
+
|
11
|
+
attr_reader :options, :source
|
12
|
+
|
13
|
+
# Intializes a new RequestLogAnalyzer::Aggregator::Base instance
|
14
|
+
# It will include the specific file format module.
|
15
|
+
def initialize(source, options = {})
|
16
|
+
@source = source
|
17
|
+
@options = options
|
18
|
+
end
|
19
|
+
|
20
|
+
# The prepare function is called just before parsing starts. This function
|
21
|
+
# can be used to initialie variables, etc.
|
22
|
+
def prepare
|
23
|
+
end
|
24
|
+
|
25
|
+
# The aggregate function is called for every request.
|
26
|
+
# Implement the aggregating functionality in this method
|
27
|
+
def aggregate(request)
|
28
|
+
end
|
29
|
+
|
30
|
+
# The finalize function is called after all sources are parsed and no more
|
31
|
+
# requests will be passed to the aggregator
|
32
|
+
def finalize
|
33
|
+
end
|
34
|
+
|
35
|
+
# The warning method is called if the parser eits a warning.
|
36
|
+
def warning(type, message, lineno)
|
37
|
+
end
|
38
|
+
|
39
|
+
# The report function is called at the end. Implement any result reporting
|
40
|
+
# in this function.
|
41
|
+
def report(output)
|
42
|
+
end
|
43
|
+
|
44
|
+
# The source_change function gets called when handling a source is started or finished.
|
45
|
+
def source_change(change, filename)
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
|
2
|
+
module RequestLogAnalyzer::Aggregator
|
3
|
+
|
4
|
+
# The database aggregator will create an SQLite3 database with all parsed request information.
|
5
|
+
#
|
6
|
+
# The prepare method will create a database schema according to the file format definitions.
|
7
|
+
# It will also create ActiveRecord::Base subclasses to interact with the created tables.
|
8
|
+
# Then, the aggregate method will be called for every parsed request. The information of
|
9
|
+
# these requests is inserted into the tables using the ActiveRecord classes.
|
10
|
+
#
|
11
|
+
# A requests table will be created, in which a record is inserted for every parsed request.
|
12
|
+
# For every line type, a separate table will be created with a request_id field to point to
|
13
|
+
# the request record, and a field for every parsed value. Finally, a warnings table will be
|
14
|
+
# created to log all parse warnings.
|
15
|
+
class DatabaseInserter < Base
|
16
|
+
|
17
|
+
attr_reader :request_count, :sources, :database
|
18
|
+
|
19
|
+
# Establishes a connection to the database and creates the necessary database schema for the
|
20
|
+
# current file format
|
21
|
+
def prepare
|
22
|
+
@sources = {}
|
23
|
+
@database = RequestLogAnalyzer::Database.new(options[:database])
|
24
|
+
@database.file_format = source.file_format
|
25
|
+
|
26
|
+
database.drop_database_schema! if options[:reset_database]
|
27
|
+
database.create_database_schema!
|
28
|
+
end
|
29
|
+
|
30
|
+
# Aggregates a request into the database
|
31
|
+
# This will create a record in the requests table and create a record for every line that has been parsed,
|
32
|
+
# in which the captured values will be stored.
|
33
|
+
def aggregate(request)
|
34
|
+
@request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
|
35
|
+
request.lines.each do |line|
|
36
|
+
class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
|
37
|
+
attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
|
38
|
+
attributes[:source_id] = @sources[line[:source]].id if @sources[line[:source]]
|
39
|
+
@request_object.send("#{line[:line_type]}_lines").build(attributes)
|
40
|
+
end
|
41
|
+
@request_object.save!
|
42
|
+
rescue SQLite3::SQLException => e
|
43
|
+
raise Interrupt, e.message
|
44
|
+
end
|
45
|
+
|
46
|
+
# Finalizes the aggregator by closing the connection to the database
|
47
|
+
def finalize
|
48
|
+
@request_count = RequestLogAnalyzer::Database::Request.count
|
49
|
+
database.disconnect
|
50
|
+
database.remove_orm_classes!
|
51
|
+
end
|
52
|
+
|
53
|
+
# Records w warining in the warnings table.
|
54
|
+
def warning(type, message, lineno)
|
55
|
+
RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
|
56
|
+
end
|
57
|
+
|
58
|
+
# Records source changes in the sources table
|
59
|
+
def source_change(change, filename)
|
60
|
+
if File.exist?(filename)
|
61
|
+
case change
|
62
|
+
when :started
|
63
|
+
@sources[filename] = RequestLogAnalyzer::Database::Source.create!(:filename => filename)
|
64
|
+
when :finished
|
65
|
+
@sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Prints a short report of what has been inserted into the database
|
71
|
+
def report(output)
|
72
|
+
output.title('Request database created')
|
73
|
+
|
74
|
+
output << "A database file has been created with all parsed request information.\n"
|
75
|
+
output << "#{@request_count} requests have been added to the database.\n"
|
76
|
+
output << "\n"
|
77
|
+
output << "To open a Ruby console to inspect the database, run the following command.\n"
|
78
|
+
output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold)
|
79
|
+
output << "\n"
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
end
|