wvanbergen-request-log-analyzer 0.3.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +2 -2
- data/TODO +0 -1
- data/bin/request-log-analyzer +28 -28
- data/bin/request-log-database +81 -0
- data/lib/base/log_parser.rb +78 -0
- data/lib/base/record_inserter.rb +139 -0
- data/lib/command_line/arguments.rb +2 -15
- data/lib/command_line/flag.rb +0 -12
- data/lib/merb_analyzer/log_parser.rb +26 -0
- data/lib/rails_analyzer/log_parser.rb +35 -0
- data/lib/rails_analyzer/record_inserter.rb +39 -0
- data/lib/request_log_analyzer.rb +161 -5
- data/tasks/test.rake +8 -0
- data/test/log_fragments/fragment_1.log +59 -0
- data/test/log_fragments/fragment_2.log +5 -0
- data/test/log_fragments/fragment_3.log +12 -0
- data/test/log_fragments/fragment_4.log +10 -0
- data/test/log_fragments/fragment_5.log +24 -0
- data/test/log_fragments/merb_1.log +84 -0
- data/test/merb_log_parser_test.rb +39 -0
- data/test/rails_log_parser_test.rb +99 -0
- data/test/record_inserter_test.rb +45 -0
- metadata +23 -16
- data/lib/request_log_analyzer/aggregator/base.rb +0 -34
- data/lib/request_log_analyzer/aggregator/database.rb +0 -86
- data/lib/request_log_analyzer/aggregator/echo.rb +0 -10
- data/lib/request_log_analyzer/aggregator/summarizer.rb +0 -53
- data/lib/request_log_analyzer/controller.rb +0 -90
- data/lib/request_log_analyzer/file_format.rb +0 -91
- data/lib/request_log_analyzer/file_format/merb.rb +0 -30
- data/lib/request_log_analyzer/file_format/rails.rb +0 -84
- data/lib/request_log_analyzer/log_parser.rb +0 -122
- data/lib/request_log_analyzer/request.rb +0 -72
- data/tasks/rspec.rake +0 -6
data/Rakefile
CHANGED
data/TODO
CHANGED
data/bin/request-log-analyzer
CHANGED
@@ -7,17 +7,14 @@ puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
|
|
7
7
|
# Parse the arguments given via commandline
|
8
8
|
begin
|
9
9
|
arguments = CommandLine::Arguments.parse do |command_line|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
command_line.
|
14
|
-
command_line.
|
15
|
-
command_line.flag(:
|
16
|
-
|
17
|
-
command_line.
|
18
|
-
command_line.switch(:colorize, :z)
|
19
|
-
#command_line.switch(:estimate_database_time, :e)
|
20
|
-
#command_line.switch(:fast, :f) #
|
10
|
+
command_line.switch(:guess_database_time, :g)
|
11
|
+
command_line.switch(:fast, :f)
|
12
|
+
command_line.switch(:colorize, :z)
|
13
|
+
command_line.switch(:merb, :m)
|
14
|
+
command_line.switch(:install, :i)
|
15
|
+
command_line.flag(:output, :alias => :o)
|
16
|
+
command_line.flag(:amount, :alias => :c)
|
17
|
+
command_line.required_files = 1
|
21
18
|
end
|
22
19
|
|
23
20
|
rescue CommandLine::Error => e
|
@@ -27,22 +24,25 @@ rescue CommandLine::Error => e
|
|
27
24
|
exit(0)
|
28
25
|
end
|
29
26
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
27
|
+
if arguments[:install]
|
28
|
+
if arguments.files.first == 'rails'
|
29
|
+
require 'ftools'
|
30
|
+
if File.directory?('./lib/tasks/')
|
31
|
+
File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
|
32
|
+
puts "Installed rake tasks."
|
33
|
+
puts "To use, run: rake log:analyze"
|
34
|
+
else
|
35
|
+
puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
|
36
|
+
puts "Installation aborted."
|
37
|
+
end
|
38
|
+
end
|
39
|
+
exit(0)
|
40
|
+
end
|
41
|
+
|
42
|
+
$colorize = true if arguments[:colorize]
|
46
43
|
|
47
44
|
# Run the request_log_analyzer!
|
48
|
-
request_log_analyzer = RequestLogAnalyzer
|
45
|
+
request_log_analyzer = RequestLogAnalyzer.new(arguments)
|
46
|
+
request_log_analyzer.analyze_this(arguments.files)
|
47
|
+
|
48
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../lib/command_line/arguments'
|
4
|
+
require File.dirname(__FILE__) + '/../lib/base/log_parser'
|
5
|
+
require File.dirname(__FILE__) + '/../lib/base/record_inserter'
|
6
|
+
require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
|
7
|
+
require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
|
8
|
+
require File.dirname(__FILE__) + '/../lib/bashcolorizer'
|
9
|
+
require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
|
10
|
+
|
11
|
+
|
12
|
+
puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
|
13
|
+
|
14
|
+
begin
|
15
|
+
|
16
|
+
$arguments = CommandLine::Arguments.parse do |command_line|
|
17
|
+
command_line.switch(:guess_database_time, :g)
|
18
|
+
command_line.switch(:reset_database, :r)
|
19
|
+
command_line.flag(:database, :alias => :d, :required => false)
|
20
|
+
command_line.required_files = 1
|
21
|
+
end
|
22
|
+
|
23
|
+
rescue CommandLine::Error => e
|
24
|
+
puts "ARGUMENT ERROR: " + e.message
|
25
|
+
puts
|
26
|
+
puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
|
27
|
+
puts
|
28
|
+
puts "Options:"
|
29
|
+
puts " --database, -t: The database file to use"
|
30
|
+
puts " --reset-database, -r: Resets the database before inserting new records"
|
31
|
+
puts " --guess-database-time, -g: Guesses the database duration of requests"
|
32
|
+
puts
|
33
|
+
puts "Examples:"
|
34
|
+
puts " ./parsetodb.rb development.log"
|
35
|
+
puts " ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"
|
36
|
+
puts
|
37
|
+
|
38
|
+
exit(0)
|
39
|
+
end
|
40
|
+
|
41
|
+
log_files = $arguments.files
|
42
|
+
db_file = $arguments[:database] || log_files.first + '.db'
|
43
|
+
|
44
|
+
if $arguments[:reset_database] && File.exist?(db_file)
|
45
|
+
File.delete(db_file)
|
46
|
+
puts "Database file cleared."
|
47
|
+
end
|
48
|
+
|
49
|
+
records_inserted = 0
|
50
|
+
inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
|
51
|
+
log_files.each do |log_file|
|
52
|
+
|
53
|
+
puts "Processing all log lines from #{log_file}..."
|
54
|
+
parser = RailsAnalyzer::LogParser.new(log_file)
|
55
|
+
|
56
|
+
pbar = ProgressBar.new(green(log_file), File.size(log_file))
|
57
|
+
parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
|
58
|
+
|
59
|
+
parser.each do |request|
|
60
|
+
db.insert(request)
|
61
|
+
records_inserted += 1
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
if $arguments[:guess_database_time]
|
66
|
+
puts "Calculating database times..."
|
67
|
+
db.calculate_db_durations!
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
started = inserter.count(:started)
|
72
|
+
completed = inserter.count(:completed)
|
73
|
+
failed = inserter.count(:failed)
|
74
|
+
|
75
|
+
puts
|
76
|
+
puts "Inserted #{records_inserted} records from #{log_files.length} files."
|
77
|
+
puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
|
78
|
+
puts
|
79
|
+
puts "Requests started: #{started}"
|
80
|
+
puts "Requests completed: #{completed}"
|
81
|
+
puts "Requests failed: #{failed}"
|
@@ -0,0 +1,78 @@
|
|
1
|
+
module Base
|
2
|
+
# Parse a log file
|
3
|
+
class LogParser
|
4
|
+
|
5
|
+
LOG_LINES = {}
|
6
|
+
|
7
|
+
# LogParser initializer
|
8
|
+
# <tt>file</tt> The fileobject this LogParser wil operate on.
|
9
|
+
def initialize(file, options = {})
|
10
|
+
@file_name = file
|
11
|
+
@options = options
|
12
|
+
@file_size = File.size(@file_name)
|
13
|
+
|
14
|
+
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
15
|
+
end
|
16
|
+
|
17
|
+
def progress(&block)
|
18
|
+
@progress_handler = block
|
19
|
+
end
|
20
|
+
|
21
|
+
# Output a warning
|
22
|
+
# <tt>message</tt> The warning message (object)
|
23
|
+
def warn(message)
|
24
|
+
puts " -> " + message.to_s
|
25
|
+
end
|
26
|
+
|
27
|
+
def convert_value(value, type)
|
28
|
+
return case type
|
29
|
+
when :string; value.to_s
|
30
|
+
when :int; value.to_i
|
31
|
+
when :sec; value.to_f
|
32
|
+
when :msec; value.to_f / 1000
|
33
|
+
when :timestamp; value.to_s # TODO: fix me?
|
34
|
+
else
|
35
|
+
warn("Unkwn type encountered: #{type}")
|
36
|
+
value
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Finds a log line and then parses the information in the line.
|
41
|
+
# Yields a hash containing the information found.
|
42
|
+
# <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
|
43
|
+
# Yeilds a Hash when it encounters a chunk of information.
|
44
|
+
def each(*line_types, &block)
|
45
|
+
log_lines_hash = self.class::LOG_LINES
|
46
|
+
|
47
|
+
|
48
|
+
# parse everything by default
|
49
|
+
line_types = log_lines_hash.keys if line_types.empty?
|
50
|
+
|
51
|
+
File.open(@file_name) do |file|
|
52
|
+
|
53
|
+
file.each_line do |line|
|
54
|
+
|
55
|
+
@progress_handler.call(file.pos, @file_size) if @progress_handler
|
56
|
+
|
57
|
+
line_types.each do |line_type|
|
58
|
+
if log_lines_hash[line_type][:teaser] =~ line
|
59
|
+
if log_lines_hash[line_type][:regexp] =~ line
|
60
|
+
|
61
|
+
captures = $~.captures
|
62
|
+
request = { :type => line_type, :line => file.lineno }
|
63
|
+
log_lines_hash[line_type][:params].each_with_index do |paramhash, index|
|
64
|
+
paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
|
65
|
+
end
|
66
|
+
|
67
|
+
yield(request) if block_given?
|
68
|
+
else
|
69
|
+
warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
@progress_handler.call(:finished, @file_size) if @progress_handler
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
module Base
|
5
|
+
|
6
|
+
# Set of functions that can be used to easily log requests into a SQLite3 Database.
|
7
|
+
class RecordInserter
|
8
|
+
|
9
|
+
attr_reader :database
|
10
|
+
attr_reader :current_request
|
11
|
+
attr_reader :warning_count
|
12
|
+
|
13
|
+
# Initializer
|
14
|
+
# <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
|
15
|
+
def initialize(db_file, options = {})
|
16
|
+
@database = SQLite3::Database.new(db_file)
|
17
|
+
@insert_statements = nil
|
18
|
+
@warning_count = 0
|
19
|
+
create_tables_if_needed!
|
20
|
+
|
21
|
+
self.initialize_hook(options) if self.respond_to?(:initialize_hook)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Calculate the database durations of the requests currenty in the database.
|
25
|
+
# Used if a logfile does contain any database durations.
|
26
|
+
def calculate_db_durations!
|
27
|
+
@database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
|
28
|
+
end
|
29
|
+
|
30
|
+
# Insert a batch of loglines into the database.
|
31
|
+
# Function prepares insert statements, yeilds and then closes and commits.
|
32
|
+
def insert_batch(&block)
|
33
|
+
@database.transaction
|
34
|
+
prepare_statements!
|
35
|
+
block.call(self)
|
36
|
+
close_prepared_statements!
|
37
|
+
@database.commit
|
38
|
+
rescue Exception => e
|
39
|
+
puts e.message
|
40
|
+
@database.rollback
|
41
|
+
end
|
42
|
+
|
43
|
+
def insert_warning(line, warning)
|
44
|
+
@database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
|
45
|
+
@warning_count += 1
|
46
|
+
end
|
47
|
+
|
48
|
+
# Insert a request into the database.
|
49
|
+
# def insert(request, close_statements = false)
|
50
|
+
# raise 'No insert defined for this log file type'
|
51
|
+
# end
|
52
|
+
|
53
|
+
# Insert a batch of files into the database.
|
54
|
+
# <tt>db_file</tt> The filename of the database file to use.
|
55
|
+
# Returns the created database.
|
56
|
+
def self.insert_batch_into(db_file, options = {}, &block)
|
57
|
+
db = self.new(db_file)
|
58
|
+
db.insert_batch(&block)
|
59
|
+
return db
|
60
|
+
end
|
61
|
+
|
62
|
+
def count(type)
|
63
|
+
@database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
|
64
|
+
end
|
65
|
+
|
66
|
+
protected
|
67
|
+
|
68
|
+
# Prepare insert statements.
|
69
|
+
def prepare_statements!
|
70
|
+
@insert_statements = {
|
71
|
+
:started => @database.prepare("
|
72
|
+
INSERT INTO started_requests ( line, timestamp, ip, method, controller, action)
|
73
|
+
VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
|
74
|
+
|
75
|
+
:failed => @database.prepare("
|
76
|
+
INSERT INTO failed_requests ( line, exception_string, stack_trace, error)
|
77
|
+
VALUES (:line, :exception_string, :stack_trace, :error)"),
|
78
|
+
|
79
|
+
:completed => @database.prepare("
|
80
|
+
INSERT INTO completed_requests ( line, url, status, duration, rendering_time, database_time)
|
81
|
+
VALUES (:line, :url, :status, :duration, :rendering, :db)")
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
# Close all prepared statments
|
86
|
+
def close_prepared_statements!
|
87
|
+
@insert_statements.each { |key, stmt| stmt.close }
|
88
|
+
end
|
89
|
+
|
90
|
+
# Create the needed database tables if they don't exist.
|
91
|
+
def create_tables_if_needed!
|
92
|
+
|
93
|
+
@database.execute("
|
94
|
+
CREATE TABLE IF NOT EXISTS started_requests (
|
95
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
96
|
+
line INTEGER NOT NULL,
|
97
|
+
timestamp DATETIME NOT NULL,
|
98
|
+
controller VARCHAR(255) NOT NULL,
|
99
|
+
action VARCHAR(255) NOT NULL,
|
100
|
+
method VARCHAR(6) NOT NULL,
|
101
|
+
ip VARCHAR(6) NOT NULL
|
102
|
+
)
|
103
|
+
");
|
104
|
+
|
105
|
+
@database.execute("
|
106
|
+
CREATE TABLE IF NOT EXISTS failed_requests (
|
107
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
108
|
+
line INTEGER NOT NULL,
|
109
|
+
started_request_id INTEGER,
|
110
|
+
error VARCHAR(255),
|
111
|
+
exception_string VARCHAR(255),
|
112
|
+
stack_trace TEXT
|
113
|
+
)
|
114
|
+
");
|
115
|
+
|
116
|
+
@database.execute("
|
117
|
+
CREATE TABLE IF NOT EXISTS completed_requests (
|
118
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
119
|
+
line INTEGER NOT NULL,
|
120
|
+
started_request_id INTEGER,
|
121
|
+
url VARCHAR(255) NOT NULL,
|
122
|
+
hashed_url VARCHAR(255),
|
123
|
+
status INTEGER NOT NULL,
|
124
|
+
duration FLOAT,
|
125
|
+
rendering_time FLOAT,
|
126
|
+
database_time FLOAT
|
127
|
+
)
|
128
|
+
");
|
129
|
+
|
130
|
+
@database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
|
131
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
132
|
+
line INTEGER NOT NULL,
|
133
|
+
warning VARCHAR(255) NOT NULL
|
134
|
+
)
|
135
|
+
");
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
139
|
+
end
|
@@ -46,7 +46,7 @@ module CommandLine
|
|
46
46
|
# <tt>flag</tt> A flag symbol like :fast
|
47
47
|
# Options
|
48
48
|
# * <tt>:expects</tt> Expects a value after the flag
|
49
|
-
def flag(flag, options
|
49
|
+
def flag(flag, options)
|
50
50
|
options[:expects] = String unless options.has_key?(:expects)
|
51
51
|
argument = Flag.new(flag, options)
|
52
52
|
@flag_definitions[argument.to_argument] = argument
|
@@ -85,14 +85,7 @@ module CommandLine
|
|
85
85
|
if flag.expects_argument?
|
86
86
|
|
87
87
|
if @arguments.length > (i + 1) && @arguments[i + 1]
|
88
|
-
|
89
|
-
if flag.multiple?
|
90
|
-
@flags[flag.name] ||= []
|
91
|
-
@flags[flag.name] << @arguments[i + 1]
|
92
|
-
else
|
93
|
-
@flags[flag.name] = @arguments[i + 1]
|
94
|
-
end
|
95
|
-
|
88
|
+
@flags[flag.name] = @arguments[i + 1]
|
96
89
|
i += 1
|
97
90
|
else
|
98
91
|
raise CommandLine::FlagExpectsArgument.new(arg)
|
@@ -122,12 +115,6 @@ module CommandLine
|
|
122
115
|
# Check if the parsed arguments meet their requirements.
|
123
116
|
# Raises CommandLineexception on error.
|
124
117
|
def check_parsed_arguments!
|
125
|
-
|
126
|
-
@flag_definitions.each do |flag, definition|
|
127
|
-
@flags[definition.name] ||= [] if definition.multiple? && !definition.default?
|
128
|
-
@flags[definition.name] ||= definition.default if definition.default?
|
129
|
-
end
|
130
|
-
|
131
118
|
if @begins_with_command && @command.nil?
|
132
119
|
raise CommandLine::CommandMissing.new
|
133
120
|
end
|
data/lib/command_line/flag.rb
CHANGED
@@ -6,8 +6,6 @@ module CommandLine
|
|
6
6
|
attr_reader :name
|
7
7
|
attr_reader :alias
|
8
8
|
attr_reader :argument
|
9
|
-
attr_reader :default
|
10
|
-
attr_reader :multiple
|
11
9
|
|
12
10
|
# Initialize new Flag
|
13
11
|
# <tt>name</tt> The name of the flag
|
@@ -17,8 +15,6 @@ module CommandLine
|
|
17
15
|
@alias = definition[:alias].to_sym if definition[:alias]
|
18
16
|
@required = definition.has_key?(:required) && definition[:required] == true
|
19
17
|
@argument = definition[:expects] if definition[:expects]
|
20
|
-
@multiple = definition[:multiple] || false
|
21
|
-
@default = definition[:default] if definition[:default]
|
22
18
|
end
|
23
19
|
|
24
20
|
# Argument representation of the flag (--fast)
|
@@ -41,14 +37,6 @@ module CommandLine
|
|
41
37
|
!@required
|
42
38
|
end
|
43
39
|
|
44
|
-
def multiple?
|
45
|
-
@multiple
|
46
|
-
end
|
47
|
-
|
48
|
-
def default?
|
49
|
-
!@default.nil?
|
50
|
-
end
|
51
|
-
|
52
40
|
# Check if flag is required
|
53
41
|
def required?
|
54
42
|
@required
|