wvanbergen-request-log-analyzer 0.3.3 → 0.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -1,5 +1,5 @@
1
1
  Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
2
2
 
3
- desc 'Default: run RSpec for request-log-analyzer.'
4
- task :default => :spec
3
+ desc 'Default: run unit tests for request-log-analyzer.'
4
+ task :default => :test
5
5
 
data/TODO CHANGED
@@ -19,7 +19,6 @@ Other:
19
19
 
20
20
 
21
21
 
22
-
23
22
  Datamining should look something like this:
24
23
 
25
24
  > request-log-analyzer myapp.log --interactive
@@ -7,17 +7,14 @@ puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
7
7
  # Parse the arguments given via commandline
8
8
  begin
9
9
  arguments = CommandLine::Arguments.parse do |command_line|
10
-
11
- #command_line.flag(:install, :alias => :i) # command_line.command(:install)
12
-
13
- command_line.flag(:format, :default => 'rails')
14
- command_line.flag(:aggregator, :alias => :a, :multiple => true)
15
- command_line.flag(:database, :alias => :d)
16
-
17
- command_line.switch(:combined_requests, :c)
18
- command_line.switch(:colorize, :z)
19
- #command_line.switch(:estimate_database_time, :e)
20
- #command_line.switch(:fast, :f) #
10
+ command_line.switch(:guess_database_time, :g)
11
+ command_line.switch(:fast, :f)
12
+ command_line.switch(:colorize, :z)
13
+ command_line.switch(:merb, :m)
14
+ command_line.switch(:install, :i)
15
+ command_line.flag(:output, :alias => :o)
16
+ command_line.flag(:amount, :alias => :c)
17
+ command_line.required_files = 1
21
18
  end
22
19
 
23
20
  rescue CommandLine::Error => e
@@ -27,22 +24,25 @@ rescue CommandLine::Error => e
27
24
  exit(0)
28
25
  end
29
26
 
30
- # if arguments[:install]
31
- # if arguments[:install] == 'rails'
32
- # require 'ftools'
33
- # if File.directory?('./lib/tasks/')
34
- # File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
35
- # puts "Installed rake tasks."
36
- # puts "To use, run: rake log:analyze"
37
- # else
38
- # puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
39
- # puts "Installation aborted."
40
- # end
41
- # else
42
- # raise "Cannot perform this install type!"
43
- # end
44
- # exit(0)
45
- # end
27
+ if arguments[:install]
28
+ if arguments.files.first == 'rails'
29
+ require 'ftools'
30
+ if File.directory?('./lib/tasks/')
31
+ File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
32
+ puts "Installed rake tasks."
33
+ puts "To use, run: rake log:analyze"
34
+ else
35
+ puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
36
+ puts "Installation aborted."
37
+ end
38
+ end
39
+ exit(0)
40
+ end
41
+
42
+ $colorize = true if arguments[:colorize]
46
43
 
47
44
  # Run the request_log_analyzer!
48
- request_log_analyzer = RequestLogAnalyzer::Controller.build(arguments).run!
45
+ request_log_analyzer = RequestLogAnalyzer.new(arguments)
46
+ request_log_analyzer.analyze_this(arguments.files)
47
+
48
+
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/ruby
2
+
3
+ require File.dirname(__FILE__) + '/../lib/command_line/arguments'
4
+ require File.dirname(__FILE__) + '/../lib/base/log_parser'
5
+ require File.dirname(__FILE__) + '/../lib/base/record_inserter'
6
+ require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
7
+ require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
8
+ require File.dirname(__FILE__) + '/../lib/bashcolorizer'
9
+ require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
10
+
11
+
12
+ puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
13
+
14
+ begin
15
+
16
+ $arguments = CommandLine::Arguments.parse do |command_line|
17
+ command_line.switch(:guess_database_time, :g)
18
+ command_line.switch(:reset_database, :r)
19
+ command_line.flag(:database, :alias => :d, :required => false)
20
+ command_line.required_files = 1
21
+ end
22
+
23
+ rescue CommandLine::Error => e
24
+ puts "ARGUMENT ERROR: " + e.message
25
+ puts
26
+ puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
27
+ puts
28
+ puts "Options:"
29
+ puts " --database, -t: The database file to use"
30
+ puts " --reset-database, -r: Resets the database before inserting new records"
31
+ puts " --guess-database-time, -g: Guesses the database duration of requests"
32
+ puts
33
+ puts "Examples:"
34
+ puts " ./parsetodb.rb development.log"
35
+ puts " ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"
36
+ puts
37
+
38
+ exit(0)
39
+ end
40
+
41
+ log_files = $arguments.files
42
+ db_file = $arguments[:database] || log_files.first + '.db'
43
+
44
+ if $arguments[:reset_database] && File.exist?(db_file)
45
+ File.delete(db_file)
46
+ puts "Database file cleared."
47
+ end
48
+
49
+ records_inserted = 0
50
+ inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
51
+ log_files.each do |log_file|
52
+
53
+ puts "Processing all log lines from #{log_file}..."
54
+ parser = RailsAnalyzer::LogParser.new(log_file)
55
+
56
+ pbar = ProgressBar.new(green(log_file), File.size(log_file))
57
+ parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
58
+
59
+ parser.each do |request|
60
+ db.insert(request)
61
+ records_inserted += 1
62
+ end
63
+ end
64
+
65
+ if $arguments[:guess_database_time]
66
+ puts "Calculating database times..."
67
+ db.calculate_db_durations!
68
+ end
69
+ end
70
+
71
+ started = inserter.count(:started)
72
+ completed = inserter.count(:completed)
73
+ failed = inserter.count(:failed)
74
+
75
+ puts
76
+ puts "Inserted #{records_inserted} records from #{log_files.length} files."
77
+ puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
78
+ puts
79
+ puts "Requests started: #{started}"
80
+ puts "Requests completed: #{completed}"
81
+ puts "Requests failed: #{failed}"
@@ -0,0 +1,78 @@
1
+ module Base
2
+ # Parse a log file
3
+ class LogParser
4
+
5
+ LOG_LINES = {}
6
+
7
+ # LogParser initializer
8
+ # <tt>file</tt> The fileobject this LogParser wil operate on.
9
+ def initialize(file, options = {})
10
+ @file_name = file
11
+ @options = options
12
+ @file_size = File.size(@file_name)
13
+
14
+ self.initialize_hook(options) if self.respond_to?(:initialize_hook)
15
+ end
16
+
17
+ def progress(&block)
18
+ @progress_handler = block
19
+ end
20
+
21
+ # Output a warning
22
+ # <tt>message</tt> The warning message (object)
23
+ def warn(message)
24
+ puts " -> " + message.to_s
25
+ end
26
+
27
+ def convert_value(value, type)
28
+ return case type
29
+ when :string; value.to_s
30
+ when :int; value.to_i
31
+ when :sec; value.to_f
32
+ when :msec; value.to_f / 1000
33
+ when :timestamp; value.to_s # TODO: fix me?
34
+ else
35
+ warn("Unkwn type encountered: #{type}")
36
+ value
37
+ end
38
+ end
39
+
40
+ # Finds a log line and then parses the information in the line.
41
+ # Yields a hash containing the information found.
42
+ # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
43
+ # Yeilds a Hash when it encounters a chunk of information.
44
+ def each(*line_types, &block)
45
+ log_lines_hash = self.class::LOG_LINES
46
+
47
+
48
+ # parse everything by default
49
+ line_types = log_lines_hash.keys if line_types.empty?
50
+
51
+ File.open(@file_name) do |file|
52
+
53
+ file.each_line do |line|
54
+
55
+ @progress_handler.call(file.pos, @file_size) if @progress_handler
56
+
57
+ line_types.each do |line_type|
58
+ if log_lines_hash[line_type][:teaser] =~ line
59
+ if log_lines_hash[line_type][:regexp] =~ line
60
+
61
+ captures = $~.captures
62
+ request = { :type => line_type, :line => file.lineno }
63
+ log_lines_hash[line_type][:params].each_with_index do |paramhash, index|
64
+ paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
65
+ end
66
+
67
+ yield(request) if block_given?
68
+ else
69
+ warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
70
+ end
71
+ end
72
+ end
73
+ end
74
+ @progress_handler.call(:finished, @file_size) if @progress_handler
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,139 @@
1
+ require 'rubygems'
2
+ require 'sqlite3'
3
+
4
+ module Base
5
+
6
+ # Set of functions that can be used to easily log requests into a SQLite3 Database.
7
+ class RecordInserter
8
+
9
+ attr_reader :database
10
+ attr_reader :current_request
11
+ attr_reader :warning_count
12
+
13
+ # Initializer
14
+ # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
15
+ def initialize(db_file, options = {})
16
+ @database = SQLite3::Database.new(db_file)
17
+ @insert_statements = nil
18
+ @warning_count = 0
19
+ create_tables_if_needed!
20
+
21
+ self.initialize_hook(options) if self.respond_to?(:initialize_hook)
22
+ end
23
+
24
+ # Calculate the database durations of the requests currenty in the database.
25
+ # Used if a logfile does contain any database durations.
26
+ def calculate_db_durations!
27
+ @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
28
+ end
29
+
30
+ # Insert a batch of loglines into the database.
31
+ # Function prepares insert statements, yeilds and then closes and commits.
32
+ def insert_batch(&block)
33
+ @database.transaction
34
+ prepare_statements!
35
+ block.call(self)
36
+ close_prepared_statements!
37
+ @database.commit
38
+ rescue Exception => e
39
+ puts e.message
40
+ @database.rollback
41
+ end
42
+
43
+ def insert_warning(line, warning)
44
+ @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
45
+ @warning_count += 1
46
+ end
47
+
48
+ # Insert a request into the database.
49
+ # def insert(request, close_statements = false)
50
+ # raise 'No insert defined for this log file type'
51
+ # end
52
+
53
+ # Insert a batch of files into the database.
54
+ # <tt>db_file</tt> The filename of the database file to use.
55
+ # Returns the created database.
56
+ def self.insert_batch_into(db_file, options = {}, &block)
57
+ db = self.new(db_file)
58
+ db.insert_batch(&block)
59
+ return db
60
+ end
61
+
62
+ def count(type)
63
+ @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
64
+ end
65
+
66
+ protected
67
+
68
+ # Prepare insert statements.
69
+ def prepare_statements!
70
+ @insert_statements = {
71
+ :started => @database.prepare("
72
+ INSERT INTO started_requests ( line, timestamp, ip, method, controller, action)
73
+ VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
74
+
75
+ :failed => @database.prepare("
76
+ INSERT INTO failed_requests ( line, exception_string, stack_trace, error)
77
+ VALUES (:line, :exception_string, :stack_trace, :error)"),
78
+
79
+ :completed => @database.prepare("
80
+ INSERT INTO completed_requests ( line, url, status, duration, rendering_time, database_time)
81
+ VALUES (:line, :url, :status, :duration, :rendering, :db)")
82
+ }
83
+ end
84
+
85
+ # Close all prepared statments
86
+ def close_prepared_statements!
87
+ @insert_statements.each { |key, stmt| stmt.close }
88
+ end
89
+
90
+ # Create the needed database tables if they don't exist.
91
+ def create_tables_if_needed!
92
+
93
+ @database.execute("
94
+ CREATE TABLE IF NOT EXISTS started_requests (
95
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
96
+ line INTEGER NOT NULL,
97
+ timestamp DATETIME NOT NULL,
98
+ controller VARCHAR(255) NOT NULL,
99
+ action VARCHAR(255) NOT NULL,
100
+ method VARCHAR(6) NOT NULL,
101
+ ip VARCHAR(6) NOT NULL
102
+ )
103
+ ");
104
+
105
+ @database.execute("
106
+ CREATE TABLE IF NOT EXISTS failed_requests (
107
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
108
+ line INTEGER NOT NULL,
109
+ started_request_id INTEGER,
110
+ error VARCHAR(255),
111
+ exception_string VARCHAR(255),
112
+ stack_trace TEXT
113
+ )
114
+ ");
115
+
116
+ @database.execute("
117
+ CREATE TABLE IF NOT EXISTS completed_requests (
118
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
119
+ line INTEGER NOT NULL,
120
+ started_request_id INTEGER,
121
+ url VARCHAR(255) NOT NULL,
122
+ hashed_url VARCHAR(255),
123
+ status INTEGER NOT NULL,
124
+ duration FLOAT,
125
+ rendering_time FLOAT,
126
+ database_time FLOAT
127
+ )
128
+ ");
129
+
130
+ @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
131
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
132
+ line INTEGER NOT NULL,
133
+ warning VARCHAR(255) NOT NULL
134
+ )
135
+ ");
136
+ end
137
+
138
+ end
139
+ end
@@ -46,7 +46,7 @@ module CommandLine
46
46
  # <tt>flag</tt> A flag symbol like :fast
47
47
  # Options
48
48
  # * <tt>:expects</tt> Expects a value after the flag
49
- def flag(flag, options = {})
49
+ def flag(flag, options)
50
50
  options[:expects] = String unless options.has_key?(:expects)
51
51
  argument = Flag.new(flag, options)
52
52
  @flag_definitions[argument.to_argument] = argument
@@ -85,14 +85,7 @@ module CommandLine
85
85
  if flag.expects_argument?
86
86
 
87
87
  if @arguments.length > (i + 1) && @arguments[i + 1]
88
-
89
- if flag.multiple?
90
- @flags[flag.name] ||= []
91
- @flags[flag.name] << @arguments[i + 1]
92
- else
93
- @flags[flag.name] = @arguments[i + 1]
94
- end
95
-
88
+ @flags[flag.name] = @arguments[i + 1]
96
89
  i += 1
97
90
  else
98
91
  raise CommandLine::FlagExpectsArgument.new(arg)
@@ -122,12 +115,6 @@ module CommandLine
122
115
  # Check if the parsed arguments meet their requirements.
123
116
  # Raises CommandLineexception on error.
124
117
  def check_parsed_arguments!
125
-
126
- @flag_definitions.each do |flag, definition|
127
- @flags[definition.name] ||= [] if definition.multiple? && !definition.default?
128
- @flags[definition.name] ||= definition.default if definition.default?
129
- end
130
-
131
118
  if @begins_with_command && @command.nil?
132
119
  raise CommandLine::CommandMissing.new
133
120
  end
@@ -6,8 +6,6 @@ module CommandLine
6
6
  attr_reader :name
7
7
  attr_reader :alias
8
8
  attr_reader :argument
9
- attr_reader :default
10
- attr_reader :multiple
11
9
 
12
10
  # Initialize new Flag
13
11
  # <tt>name</tt> The name of the flag
@@ -17,8 +15,6 @@ module CommandLine
17
15
  @alias = definition[:alias].to_sym if definition[:alias]
18
16
  @required = definition.has_key?(:required) && definition[:required] == true
19
17
  @argument = definition[:expects] if definition[:expects]
20
- @multiple = definition[:multiple] || false
21
- @default = definition[:default] if definition[:default]
22
18
  end
23
19
 
24
20
  # Argument representation of the flag (--fast)
@@ -41,14 +37,6 @@ module CommandLine
41
37
  !@required
42
38
  end
43
39
 
44
- def multiple?
45
- @multiple
46
- end
47
-
48
- def default?
49
- !@default.nil?
50
- end
51
-
52
40
  # Check if flag is required
53
41
  def required?
54
42
  @required