request-log-analyzer 1.12.4 → 1.12.5
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +12 -4
- data/README.rdoc +1 -0
- data/bin/request-log-analyzer +2 -2
- data/lib/cli/command_line_arguments.rb +3 -3
- data/lib/cli/database_console_init.rb +1 -1
- data/lib/cli/progressbar.rb +5 -4
- data/lib/cli/tools.rb +48 -44
- data/lib/ordered_hash.rb +167 -0
- data/lib/request_log_analyzer.rb +1 -0
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +1 -1
- data/lib/request_log_analyzer/aggregator/echo.rb +1 -1
- data/lib/request_log_analyzer/aggregator/summarizer.rb +9 -6
- data/lib/request_log_analyzer/database/base.rb +3 -5
- data/lib/request_log_analyzer/file_format.rb +3 -3
- data/lib/request_log_analyzer/file_format/mysql.rb +1 -1
- data/lib/request_log_analyzer/file_format/oink.rb +28 -28
- data/lib/request_log_analyzer/file_format/postgresql.rb +1 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +1 -4
- data/lib/request_log_analyzer/line_definition.rb +4 -2
- data/lib/request_log_analyzer/log_processor.rb +1 -1
- data/lib/request_log_analyzer/output/fixed_width.rb +0 -1
- data/lib/request_log_analyzer/source/log_parser.rb +13 -8
- data/lib/request_log_analyzer/tracker/numeric_value.rb +3 -3
- data/request-log-analyzer.gemspec +3 -3
- data/spec/unit/database/base_class_spec.rb +13 -22
- metadata +83 -77
data/.travis.yml
CHANGED
@@ -4,11 +4,19 @@ rvm:
|
|
4
4
|
- 1.8.7
|
5
5
|
- 1.9.2
|
6
6
|
- 1.9.3
|
7
|
+
- ruby-head
|
7
8
|
- ree
|
8
9
|
- jruby-18mode
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
- jruby-19mode
|
11
|
+
- jruby-head
|
12
|
+
- rbx-18mode
|
13
|
+
- rbx-19mode
|
14
|
+
matrix:
|
15
|
+
allow_failures:
|
16
|
+
- rvm: jruby-19mode
|
17
|
+
- rvm: rbx-19mode
|
18
|
+
- rvm: jruby-head
|
19
|
+
- rvm: ruby-head
|
12
20
|
notifications:
|
13
21
|
email:
|
14
|
-
- info@railsdoctors.com
|
22
|
+
- info@railsdoctors.com
|
data/README.rdoc
CHANGED
@@ -7,6 +7,7 @@ This is a simple command line tool to analyze request log files in various forma
|
|
7
7
|
* Uses several metrics, including cumulative request time, mean request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://github.com/wvanbergen/request-log-analyzer/wiki/sample-output)
|
8
8
|
* Low memory footprint and reasonably fast, so it is safe to run on a production server.
|
9
9
|
* MIT licensed
|
10
|
+
* Runs on all rubies {<img src="https://secure.travis-ci.org/wvanbergen/request-log-analyzer.png" />}[http://travis-ci.org/wvanbergen/request-log-analyzer]
|
10
11
|
|
11
12
|
See the project wiki at http://github.com/wvanbergen/request-log-analyzer/wiki for documentation and additional information.
|
12
13
|
|
data/bin/request-log-analyzer
CHANGED
@@ -54,7 +54,7 @@ begin
|
|
54
54
|
|
55
55
|
command_line.switch(:boring, :b)
|
56
56
|
command_line.option(:output, :alias => :o, :default => 'fixedwidth')
|
57
|
-
command_line.option(:report_width, :default => terminal_width - 1)
|
57
|
+
command_line.option(:report_width, :default => CommandLine::Tools.terminal_width - 1)
|
58
58
|
command_line.option(:report_amount, :default => 20)
|
59
59
|
command_line.option(:report_sort, :default => 'sum,mean')
|
60
60
|
|
@@ -111,7 +111,7 @@ end
|
|
111
111
|
|
112
112
|
case arguments.command
|
113
113
|
when :install
|
114
|
-
install_rake_tasks(arguments.parameters[0])
|
114
|
+
CommandLine::Tools.install_rake_tasks(arguments.parameters[0])
|
115
115
|
when :console
|
116
116
|
require 'cli/database_console'
|
117
117
|
DatabaseConsole.new(arguments).run!
|
@@ -17,7 +17,7 @@ module CommandLine
|
|
17
17
|
# <tt>definition</tt> The definition of the flag.
|
18
18
|
def initialize(name, definition = {})
|
19
19
|
@name = CommandLine::Option.rewrite(name)
|
20
|
-
@alias = definition[:alias]
|
20
|
+
@alias = definition[:alias] ? definition[:alias].to_sym : nil
|
21
21
|
@required = definition.has_key?(:required) && definition[:required] == true
|
22
22
|
@parameter_count = definition[:parameters] || 1
|
23
23
|
@multiple = definition[:multiple] || false
|
@@ -103,7 +103,7 @@ module CommandLine
|
|
103
103
|
|
104
104
|
def [](option_name)
|
105
105
|
option_symbol = CommandLine::Option.rewrite(option_name)
|
106
|
-
if the_option = @options.detect { |(
|
106
|
+
if the_option = @options.detect { |(_, odef)| odef =~ option_symbol }
|
107
107
|
the_option[1]
|
108
108
|
else
|
109
109
|
raise CommandLine::UnknownOption, option_name
|
@@ -164,7 +164,7 @@ module CommandLine
|
|
164
164
|
end
|
165
165
|
|
166
166
|
def [](option)
|
167
|
-
if the_option = @options.detect { |(key,
|
167
|
+
if the_option = @options.detect { |(key, _)| key =~ option }
|
168
168
|
the_option[1]
|
169
169
|
else
|
170
170
|
@current_definition[option].default_value
|
@@ -30,7 +30,7 @@ class Request
|
|
30
30
|
inspect_line = " - #{line.line_type} (line #{line.lineno})"
|
31
31
|
if (inspect_attributes = line.attributes.reject { |(k, v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any?
|
32
32
|
inspect_attributes = inspect_attributes.map { |(k,v)| "#{k} = #{v.inspect}" }.join(', ')
|
33
|
-
inspect_line << "\n " + wordwrap(inspect_attributes, terminal_width - 6, " ")
|
33
|
+
inspect_line << "\n " + wordwrap(inspect_attributes, CommandLine::Tools.terminal_width - 6, " ")
|
34
34
|
end
|
35
35
|
inspect_line
|
36
36
|
end
|
data/lib/cli/progressbar.rb
CHANGED
@@ -29,6 +29,7 @@ module CommandLine
|
|
29
29
|
clear
|
30
30
|
show
|
31
31
|
end
|
32
|
+
|
32
33
|
attr_reader :title
|
33
34
|
attr_reader :current
|
34
35
|
attr_reader :total
|
@@ -120,13 +121,13 @@ module CommandLine
|
|
120
121
|
end
|
121
122
|
|
122
123
|
def show
|
123
|
-
arguments = @format_arguments.map
|
124
|
+
arguments = @format_arguments.map do |method|
|
124
125
|
method = sprintf("fmt_%s", method)
|
125
126
|
send(method)
|
126
|
-
|
127
|
+
end
|
127
128
|
line = sprintf(@format, *arguments)
|
128
129
|
|
129
|
-
width = terminal_width(80)
|
130
|
+
width = CommandLine::Tools.terminal_width(80, @out)
|
130
131
|
if line.length == width - 1
|
131
132
|
@out.print(line + eol)
|
132
133
|
@out.flush
|
@@ -159,7 +160,7 @@ module CommandLine
|
|
159
160
|
public
|
160
161
|
def clear
|
161
162
|
@out.print "\r"
|
162
|
-
@out.print(" " * (terminal_width(80) - 1))
|
163
|
+
@out.print(" " * (CommandLine::Tools.terminal_width(80) - 1))
|
163
164
|
@out.print "\r"
|
164
165
|
end
|
165
166
|
|
data/lib/cli/tools.rb
CHANGED
@@ -1,50 +1,54 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
1
|
+
module CommandLine
|
2
|
+
module Tools
|
3
|
+
extend self
|
4
|
+
|
5
|
+
# Try to determine the terminal with.
|
6
|
+
# If it is not possible to to so, it returns the default_width.
|
7
|
+
# <tt>default_width</tt> Defaults to 81
|
8
|
+
def terminal_width(default_width = 81, out = STDOUT)
|
9
|
+
|
10
|
+
begin
|
11
|
+
tiocgwinsz = 0x5413
|
12
|
+
data = [0, 0, 0, 0].pack("SSSS")
|
13
|
+
if !RUBY_PLATFORM.include?('java') && out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl
|
14
|
+
_, cols, _, _ = data.unpack("SSSS")
|
15
|
+
raise unless cols > 0
|
16
|
+
cols
|
17
|
+
else
|
18
|
+
raise
|
19
|
+
end
|
20
|
+
rescue
|
21
|
+
begin
|
22
|
+
IO.popen('stty -a 2>&1') do |pipe|
|
23
|
+
column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
|
24
|
+
raise unless column_line
|
25
|
+
$1.to_i
|
26
|
+
end
|
27
|
+
rescue
|
28
|
+
default_width
|
29
|
+
end
|
22
30
|
end
|
23
|
-
rescue
|
24
|
-
default_width
|
25
31
|
end
|
26
|
-
end
|
27
|
-
end
|
28
32
|
|
29
|
-
# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
|
30
|
-
# environment integration.
|
31
|
-
# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
|
32
|
-
# Raises if it cannot find the project folder or if the install_type is now known.
|
33
|
-
def install_rake_tasks(install_type = :rails)
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
33
|
+
# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
|
34
|
+
# environment integration.
|
35
|
+
# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
|
36
|
+
# Raises if it cannot find the project folder or if the install_type is now known.
|
37
|
+
def install_rake_tasks(install_type = :rails)
|
38
|
+
if install_type.to_sym == :rails
|
39
|
+
require 'fileutils'
|
40
|
+
if File.directory?('./lib/tasks/')
|
41
|
+
task_file = File.expand_path('../../tasks/request_log_analyzer.rake', File.dirname(__FILE__))
|
42
|
+
FileUtils.copy(task_file, './lib/tasks/request_log_analyze.rake')
|
43
|
+
puts "Installed rake tasks."
|
44
|
+
puts "To use, run: rake rla:report"
|
45
|
+
else
|
46
|
+
puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
|
47
|
+
puts "Installation aborted."
|
48
|
+
end
|
49
|
+
else
|
50
|
+
raise "Cannot perform this install type! (#{install_type.to_s})"
|
51
|
+
end
|
44
52
|
end
|
45
|
-
else
|
46
|
-
raise "Cannot perform this install type! (#{install_type.to_s})"
|
47
53
|
end
|
48
54
|
end
|
49
|
-
|
50
|
-
|
data/lib/ordered_hash.rb
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
if RUBY_VERSION.to_f <= 1.8
|
2
|
+
|
3
|
+
# Simplified version copied from Activesupport::OrderedHash
|
4
|
+
class OrderedHash < ::Hash #:nodoc:
|
5
|
+
# In MRI the Hash class is core and written in C. In particular, methods are
|
6
|
+
# programmed with explicit C function calls and polymorphism is not honored.
|
7
|
+
#
|
8
|
+
# For example, []= is crucial in this implementation to maintain the @keys
|
9
|
+
# array but hash.c invokes rb_hash_aset() originally. This prevents method
|
10
|
+
# reuse through inheritance and forces us to reimplement stuff.
|
11
|
+
#
|
12
|
+
# For instance, we cannot use the inherited #merge! because albeit the algorithm
|
13
|
+
# itself would work, our []= is not being called at all by the C code.
|
14
|
+
|
15
|
+
def initialize(*args, &block)
|
16
|
+
super
|
17
|
+
@keys = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.[](*args)
|
21
|
+
ordered_hash = new
|
22
|
+
|
23
|
+
if (args.length == 1 && args.first.is_a?(Array))
|
24
|
+
args.first.each do |key_value_pair|
|
25
|
+
next unless (key_value_pair.is_a?(Array))
|
26
|
+
ordered_hash[key_value_pair[0]] = key_value_pair[1]
|
27
|
+
end
|
28
|
+
|
29
|
+
return ordered_hash
|
30
|
+
end
|
31
|
+
|
32
|
+
unless (args.size % 2 == 0)
|
33
|
+
raise ArgumentError.new("odd number of arguments for Hash")
|
34
|
+
end
|
35
|
+
|
36
|
+
args.each_with_index do |val, ind|
|
37
|
+
next if (ind % 2 != 0)
|
38
|
+
ordered_hash[val] = args[ind + 1]
|
39
|
+
end
|
40
|
+
|
41
|
+
ordered_hash
|
42
|
+
end
|
43
|
+
|
44
|
+
def initialize_copy(other)
|
45
|
+
super
|
46
|
+
# make a deep copy of keys
|
47
|
+
@keys = other.keys
|
48
|
+
end
|
49
|
+
|
50
|
+
def []=(key, value)
|
51
|
+
@keys << key unless has_key?(key)
|
52
|
+
super
|
53
|
+
end
|
54
|
+
|
55
|
+
def delete(key)
|
56
|
+
if has_key? key
|
57
|
+
index = @keys.index(key)
|
58
|
+
@keys.delete_at index
|
59
|
+
end
|
60
|
+
super
|
61
|
+
end
|
62
|
+
|
63
|
+
def delete_if
|
64
|
+
super
|
65
|
+
sync_keys!
|
66
|
+
self
|
67
|
+
end
|
68
|
+
|
69
|
+
def reject!
|
70
|
+
super
|
71
|
+
sync_keys!
|
72
|
+
self
|
73
|
+
end
|
74
|
+
|
75
|
+
def reject(&block)
|
76
|
+
dup.reject!(&block)
|
77
|
+
end
|
78
|
+
|
79
|
+
def keys
|
80
|
+
@keys.dup
|
81
|
+
end
|
82
|
+
|
83
|
+
def values
|
84
|
+
@keys.collect { |key| self[key] }
|
85
|
+
end
|
86
|
+
|
87
|
+
def to_hash
|
88
|
+
self
|
89
|
+
end
|
90
|
+
|
91
|
+
def to_a
|
92
|
+
@keys.map { |key| [ key, self[key] ] }
|
93
|
+
end
|
94
|
+
|
95
|
+
def each_key
|
96
|
+
return to_enum(:each_key) unless block_given?
|
97
|
+
@keys.each { |key| yield key }
|
98
|
+
self
|
99
|
+
end
|
100
|
+
|
101
|
+
def each_value
|
102
|
+
return to_enum(:each_value) unless block_given?
|
103
|
+
@keys.each { |key| yield self[key]}
|
104
|
+
self
|
105
|
+
end
|
106
|
+
|
107
|
+
def each
|
108
|
+
return to_enum(:each) unless block_given?
|
109
|
+
@keys.each {|key| yield [key, self[key]]}
|
110
|
+
self
|
111
|
+
end
|
112
|
+
|
113
|
+
alias_method :each_pair, :each
|
114
|
+
|
115
|
+
alias_method :select, :find_all
|
116
|
+
|
117
|
+
def clear
|
118
|
+
super
|
119
|
+
@keys.clear
|
120
|
+
self
|
121
|
+
end
|
122
|
+
|
123
|
+
def shift
|
124
|
+
k = @keys.first
|
125
|
+
v = delete(k)
|
126
|
+
[k, v]
|
127
|
+
end
|
128
|
+
|
129
|
+
def merge!(other_hash)
|
130
|
+
if block_given?
|
131
|
+
other_hash.each { |k, v| self[k] = key?(k) ? yield(k, self[k], v) : v }
|
132
|
+
else
|
133
|
+
other_hash.each { |k, v| self[k] = v }
|
134
|
+
end
|
135
|
+
self
|
136
|
+
end
|
137
|
+
|
138
|
+
alias_method :update, :merge!
|
139
|
+
|
140
|
+
def merge(other_hash, &block)
|
141
|
+
dup.merge!(other_hash, &block)
|
142
|
+
end
|
143
|
+
|
144
|
+
# When replacing with another hash, the initial order of our keys must come from the other hash -ordered or not.
|
145
|
+
def replace(other)
|
146
|
+
super
|
147
|
+
@keys = other.keys
|
148
|
+
self
|
149
|
+
end
|
150
|
+
|
151
|
+
def invert
|
152
|
+
OrderedHash[self.to_a.map!{|key_value_pair| key_value_pair.reverse}]
|
153
|
+
end
|
154
|
+
|
155
|
+
def inspect
|
156
|
+
"#<OrderedHash #{super}>"
|
157
|
+
end
|
158
|
+
|
159
|
+
private
|
160
|
+
def sync_keys!
|
161
|
+
@keys.delete_if {|k| !has_key?(k)}
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
else
|
166
|
+
::OrderedHash = ::Hash
|
167
|
+
end
|
data/lib/request_log_analyzer.rb
CHANGED
@@ -35,7 +35,7 @@ module RequestLogAnalyzer::Aggregator
|
|
35
35
|
@request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
|
36
36
|
request.lines.each do |line|
|
37
37
|
class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
|
38
|
-
attributes = Hash[*line.select { |(
|
38
|
+
attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s)}.flatten]
|
39
39
|
|
40
40
|
# Fix encoding patch for 1.9.2
|
41
41
|
attributes.each do |k,v|
|
@@ -12,7 +12,7 @@ module RequestLogAnalyzer::Aggregator
|
|
12
12
|
# Display every parsed line immediately to the terminal
|
13
13
|
def aggregate(request)
|
14
14
|
puts "\nRequest: \n" + request.lines.map { |l|
|
15
|
-
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k,
|
15
|
+
"\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k,_)| [:lineno, :line_type].include?(k) }.inspect}" }.join("\n")
|
16
16
|
end
|
17
17
|
|
18
18
|
# Capture all warnings during parsing
|
@@ -26,12 +26,12 @@ module RequestLogAnalyzer::Aggregator
|
|
26
26
|
def method_missing(tracker_method, *args)
|
27
27
|
track(tracker_method, *args)
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
# Helper function to initialize a tracker and add it to the tracker array.
|
31
31
|
# <tt>tracker_class</tt> The class to include
|
32
32
|
# <tt>optiont</tt> The options to pass to the trackers.
|
33
33
|
def track(tracker_klass, value_field = {}, other_options = {})
|
34
|
-
options = value_field.kind_of?(Symbol) ? other_options.merge(:value => value_field) : value_field.merge(other_options)
|
34
|
+
options = value_field.kind_of?(Symbol) ? other_options.merge(:value => value_field) : value_field.merge(other_options)
|
35
35
|
tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer.to_camelcase(tracker_klass)) if tracker_klass.kind_of?(Symbol)
|
36
36
|
@trackers << tracker_klass.new(options)
|
37
37
|
end
|
@@ -108,9 +108,12 @@ module RequestLogAnalyzer::Aggregator
|
|
108
108
|
|
109
109
|
output.with_style(:cell_separator => false) do
|
110
110
|
output.table({:width => 20}, {:font => :bold}) do |rows|
|
111
|
-
|
112
|
-
|
113
|
-
|
111
|
+
source.processed_files.each do |f|
|
112
|
+
rows << ['Processsed File:', f]
|
113
|
+
end
|
114
|
+
rows << ['Parsed lines:', source.parsed_lines]
|
115
|
+
rows << ['Skipped lines:', source.skipped_lines]
|
116
|
+
rows << ['Parsed requests:', source.parsed_requests]
|
114
117
|
rows << ['Skipped requests:', source.skipped_requests]
|
115
118
|
rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
|
116
119
|
end
|
@@ -134,7 +137,7 @@ module RequestLogAnalyzer::Aggregator
|
|
134
137
|
|
135
138
|
# Returns true if there were any warnings generated by the trackers
|
136
139
|
def has_warnings?
|
137
|
-
@warnings_encountered.inject(0) { |result, (
|
140
|
+
@warnings_encountered.inject(0) { |result, (_, value)| result += value } > 0
|
138
141
|
end
|
139
142
|
|
140
143
|
# Returns true if there were any log ordering warnings
|
@@ -26,8 +26,7 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
26
26
|
class_attribute :line_definition
|
27
27
|
cattr_accessor :database
|
28
28
|
|
29
|
-
def self.subclass_from_line_definition(definition)
|
30
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
29
|
+
def self.subclass_from_line_definition(definition, klass = Class.new(RequestLogAnalyzer::Database::Base))
|
31
30
|
klass.table_name = "#{definition.name}_lines"
|
32
31
|
|
33
32
|
klass.line_definition = definition
|
@@ -47,10 +46,9 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
|
|
47
46
|
return klass
|
48
47
|
end
|
49
48
|
|
50
|
-
def self.subclass_from_table(table)
|
49
|
+
def self.subclass_from_table(table, klass = Class.new(RequestLogAnalyzer::Database::Base))
|
51
50
|
raise "Table #{table} not found!" unless database.connection.table_exists?(table)
|
52
|
-
|
53
|
-
klass = Class.new(RequestLogAnalyzer::Database::Base)
|
51
|
+
|
54
52
|
klass.table_name = table
|
55
53
|
|
56
54
|
if klass.column_names.include?('request_id')
|
@@ -258,7 +258,7 @@ module RequestLogAnalyzer::FileFormat
|
|
258
258
|
return self.new(line_definer.line_definitions, report_definer.trackers)
|
259
259
|
end
|
260
260
|
|
261
|
-
def initialize(line_definitions =
|
261
|
+
def initialize(line_definitions = OrderedHash.new, report_trackers = [])
|
262
262
|
@line_definitions, @report_trackers = line_definitions, report_trackers
|
263
263
|
end
|
264
264
|
|
@@ -287,7 +287,7 @@ module RequestLogAnalyzer::FileFormat
|
|
287
287
|
# Checks whether the line definitions form a valid language.
|
288
288
|
# A file format should have at least a header and a footer line type
|
289
289
|
def valid_line_definitions?
|
290
|
-
line_definitions.any? { |(
|
290
|
+
line_definitions.any? { |(_, ld)| ld.header } && line_definitions.any? { |(_, ld)| ld.footer }
|
291
291
|
end
|
292
292
|
|
293
293
|
# Checks whether the request class inherits from the base Request class.
|
@@ -297,7 +297,7 @@ module RequestLogAnalyzer::FileFormat
|
|
297
297
|
|
298
298
|
# Returns true if this language captures the given symbol in one of its line definitions
|
299
299
|
def captures?(name)
|
300
|
-
line_definitions.any? { |(
|
300
|
+
line_definitions.any? { |(_, ld)| ld.captures?(name) }
|
301
301
|
end
|
302
302
|
|
303
303
|
# Function that a file format con implement to monkey patch the environment.
|
@@ -93,7 +93,7 @@ module RequestLogAnalyzer::FileFormat
|
|
93
93
|
|
94
94
|
# Convert the timestamp to an integer
|
95
95
|
def convert_timestamp(value, definition)
|
96
|
-
|
96
|
+
_, y, m, d, h, i, s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
97
97
|
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
98
98
|
end
|
99
99
|
end
|
@@ -43,27 +43,27 @@ class RequestLogAnalyzer::FileFormat::Oink < RequestLogAnalyzer::FileFormat::Rai
|
|
43
43
|
end
|
44
44
|
|
45
45
|
class Request < RequestLogAnalyzer::FileFormat::Rails::Request
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
46
|
+
# Overrides the #validate method to handle PID updating.
|
47
|
+
def validate
|
48
|
+
update_pids
|
49
|
+
super
|
50
|
+
end
|
51
51
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
52
|
+
# Accessor for memory information associated with the specified request PID. If no memory exists
|
53
|
+
# for this request's :pid, the memory tracking is initialized.
|
54
|
+
def pid_memory
|
55
|
+
file_format.pids[self[:pid]] ||= { :last_memory_reading => -1, :current_memory_reading => -1 }
|
56
|
+
end
|
57
57
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
58
|
+
# Calculates :memory_diff for each request based on the last completed request that was not a failure.
|
59
|
+
def update_pids
|
60
|
+
# memory isn't recorded with exceptions. need to set #last_memory_reading+ to -1 as
|
61
|
+
# the memory used could have changed. for the next request the memory change will not be recorded.
|
62
|
+
#
|
63
|
+
# NOTE - the failure regex was not matching with a Rails Development log file.
|
64
|
+
if has_line_type?(:failure) and processing = has_line_type?(:processing)
|
65
|
+
pid_memory[:last_memory_reading] = -1
|
66
|
+
elsif mem_line = has_line_type?(:memory_usage)
|
67
67
|
memory_reading = mem_line[:memory]
|
68
68
|
pid_memory[:current_memory_reading] = memory_reading
|
69
69
|
# calcuate the change in memory
|
@@ -79,18 +79,18 @@ class RequestLogAnalyzer::FileFormat::Oink < RequestLogAnalyzer::FileFormat::Rai
|
|
79
79
|
pid_memory[:current_memory_reading] = -1
|
80
80
|
end # if mem_line
|
81
81
|
return true
|
82
|
-
|
82
|
+
end
|
83
83
|
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
84
|
+
def convert_pipe_separated_counts(value, capture_definition)
|
85
|
+
count_strings = value.split(' | ')
|
86
|
+
count_arrays = count_strings.map do |count_string|
|
87
|
+
if count_string =~ /^(\w+): (\d+)/
|
88
|
+
[$1, $2.to_i]
|
89
|
+
end
|
89
90
|
end
|
90
|
-
end
|
91
91
|
|
92
|
-
|
93
|
-
|
92
|
+
Hash[count_arrays]
|
93
|
+
end
|
94
94
|
|
95
95
|
end # class Request
|
96
96
|
end
|
@@ -61,7 +61,7 @@ module RequestLogAnalyzer::FileFormat
|
|
61
61
|
|
62
62
|
# Convert the timestamp to an integer
|
63
63
|
def convert_timestamp(value, definition)
|
64
|
-
|
64
|
+
_, y, m, d, h, i, s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
65
65
|
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
66
66
|
end
|
67
67
|
end
|
@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
|
|
8
8
|
line_definition :access do |line|
|
9
9
|
line.header = true
|
10
10
|
line.footer = true
|
11
|
-
line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) (
|
11
|
+
line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) (\S+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/
|
12
12
|
|
13
13
|
line.capture(:timestamp).as(:timestamp)
|
14
14
|
line.capture(:remote_ip)
|
@@ -49,8 +49,5 @@ module RequestLogAnalyzer::FileFormat
|
|
49
49
|
"#{value[0,4]}#{value[5,2]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
|
50
50
|
end
|
51
51
|
end
|
52
|
-
|
53
|
-
|
54
52
|
end
|
55
|
-
|
56
53
|
end
|
@@ -10,7 +10,7 @@ module RequestLogAnalyzer
|
|
10
10
|
attr_accessor :line_definitions
|
11
11
|
|
12
12
|
def initialize
|
13
|
-
@line_definitions =
|
13
|
+
@line_definitions = OrderedHash.new
|
14
14
|
end
|
15
15
|
|
16
16
|
def initialize_copy(other)
|
@@ -66,7 +66,9 @@ module RequestLogAnalyzer
|
|
66
66
|
end
|
67
67
|
|
68
68
|
def capture(name)
|
69
|
-
new_capture_hash =
|
69
|
+
new_capture_hash = OrderedHash.new()
|
70
|
+
new_capture_hash[:name] = name
|
71
|
+
new_capture_hash[:type] = :string
|
70
72
|
captures << new_capture_hash
|
71
73
|
CaptureDefiner.new(new_capture_hash)
|
72
74
|
end
|
@@ -52,7 +52,7 @@ module RequestLogAnalyzer
|
|
52
52
|
# in which the actual processing is performed.
|
53
53
|
# <tt>file</tt> The file to process
|
54
54
|
def process_file(file)
|
55
|
-
File.open(file, 'r') { |
|
55
|
+
File.open(file, 'r') { |io| process_io(io) }
|
56
56
|
end
|
57
57
|
|
58
58
|
# Processes an input stream by iteration over each line and processing it according to
|
@@ -164,7 +164,6 @@ module RequestLogAnalyzer::Output
|
|
164
164
|
end
|
165
165
|
|
166
166
|
if column_widths.include?(nil)
|
167
|
-
fill_column = columns[column_widths.index(nil)]
|
168
167
|
width_left = options[:width] - ((columns.length - 1) * (style[:cell_separator] ? 3 : 1)) - column_widths.compact.inject(0) { |sum, col| sum + col}
|
169
168
|
column_widths[column_widths.index(nil)] = width_left
|
170
169
|
end
|
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::Source
|
|
15
15
|
|
16
16
|
# The maximum number of bytes to read from a line.
|
17
17
|
DEFAULT_MAX_LINE_LENGTH = 8096
|
18
|
-
|
18
|
+
|
19
19
|
DEFAULT_LINE_DIVIDER = "\n"
|
20
20
|
|
21
21
|
# The default parse strategy that will be used to parse the input.
|
@@ -24,7 +24,7 @@ module RequestLogAnalyzer::Source
|
|
24
24
|
# All available parse strategies.
|
25
25
|
PARSE_STRATEGIES = ['cautious', 'assume-correct']
|
26
26
|
|
27
|
-
attr_reader :source_files, :current_file, :current_lineno
|
27
|
+
attr_reader :source_files, :current_file, :current_lineno, :processed_files
|
28
28
|
attr_reader :warnings, :parsed_lines, :parsed_requests, :skipped_lines, :skipped_requests
|
29
29
|
|
30
30
|
# Initializes the log file parser instance.
|
@@ -44,8 +44,10 @@ module RequestLogAnalyzer::Source
|
|
44
44
|
@current_source = nil
|
45
45
|
@current_file = nil
|
46
46
|
@current_lineno = nil
|
47
|
+
@processed_files = []
|
47
48
|
@source_files = options[:source_files]
|
48
49
|
@progress_handler = nil
|
50
|
+
@warning_handler = nil
|
49
51
|
|
50
52
|
@options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
|
51
53
|
unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
|
@@ -56,7 +58,7 @@ module RequestLogAnalyzer::Source
|
|
56
58
|
def max_line_length
|
57
59
|
file_format.max_line_length || DEFAULT_MAX_LINE_LENGTH
|
58
60
|
end
|
59
|
-
|
61
|
+
|
60
62
|
def line_divider
|
61
63
|
file_format.line_divider || DEFAULT_LINE_DIVIDER
|
62
64
|
end
|
@@ -128,6 +130,9 @@ module RequestLogAnalyzer::Source
|
|
128
130
|
|
129
131
|
@progress_handler.call(:finished, file) if @progress_handler
|
130
132
|
@progress_handler = nil
|
133
|
+
|
134
|
+
@processed_files.push(@current_source.dup)
|
135
|
+
|
131
136
|
else
|
132
137
|
IO.popen(decompress_file?(file), 'rb') { |f| parse_io(f, options, &block) }
|
133
138
|
end
|
@@ -145,7 +150,7 @@ module RequestLogAnalyzer::Source
|
|
145
150
|
def parse_stream(stream, options = {}, &block)
|
146
151
|
parse_io(stream, options, &block)
|
147
152
|
end
|
148
|
-
|
153
|
+
|
149
154
|
# Parses a string. It will simply call parse_io. This function does not support progress updates.
|
150
155
|
# <tt>string</tt>:: The string that should be parsed.
|
151
156
|
# <tt>options</tt>:: A Hash of options that will be pased to parse_io.
|
@@ -180,7 +185,7 @@ module RequestLogAnalyzer::Source
|
|
180
185
|
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
181
186
|
@current_lineno = nil
|
182
187
|
end
|
183
|
-
|
188
|
+
|
184
189
|
# This method loops over each line of the input stream. It will try to parse this line as any of
|
185
190
|
# the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
|
186
191
|
# It will then combine these parsed line into requests using heuristics. These requests (see
|
@@ -194,7 +199,7 @@ module RequestLogAnalyzer::Source
|
|
194
199
|
# This is a Ruby 1.8 specific version that doesn't offer memory protection.
|
195
200
|
#
|
196
201
|
# <tt>io</tt>:: The IO instance to use as source
|
197
|
-
# <tt>options</tt>:: A hash of options that can be used by the parser.
|
202
|
+
# <tt>options</tt>:: A hash of options that can be used by the parser.
|
198
203
|
def parse_io_18(io, options = {}, &block) # :yields: request
|
199
204
|
@line_divider = options[:line_divider] || line_divider
|
200
205
|
@current_lineno = 0
|
@@ -207,9 +212,9 @@ module RequestLogAnalyzer::Source
|
|
207
212
|
warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
|
208
213
|
@current_lineno = nil
|
209
214
|
end
|
210
|
-
|
215
|
+
|
211
216
|
alias_method :parse_io, RUBY_VERSION.to_f < 1.9 ? :parse_io_18 : :parse_io_19
|
212
|
-
|
217
|
+
|
213
218
|
# Parses a single line using the current file format. If successful, use the parsed
|
214
219
|
# information to build a request
|
215
220
|
# <tt>line</tt>:: The line to parse
|
@@ -68,7 +68,7 @@ module RequestLogAnalyzer::Tracker
|
|
68
68
|
top_categories = output.slice_results(sorted_by(sort))
|
69
69
|
output.with_style(:top_line => true) do
|
70
70
|
output.table(*statistics_header(:title => options[:title], :highlight => sort)) do |rows|
|
71
|
-
top_categories.each { |(
|
71
|
+
top_categories.each { |(category, _)| rows << statistics_row(category) }
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
@@ -292,12 +292,12 @@ module RequestLogAnalyzer::Tracker
|
|
292
292
|
|
293
293
|
# Get the cumlative duration of a all categories.
|
294
294
|
def sum_overall
|
295
|
-
@categories.inject(0.0) { |sum, (
|
295
|
+
@categories.inject(0.0) { |sum, (_, cat)| sum + cat[:sum] }
|
296
296
|
end
|
297
297
|
|
298
298
|
# Get the total hits of a all categories.
|
299
299
|
def hits_overall
|
300
|
-
@categories.inject(0) { |sum, (
|
300
|
+
@categories.inject(0) { |sum, (_, cat)| sum + cat[:hits] }
|
301
301
|
end
|
302
302
|
|
303
303
|
# Return categories sorted by a given key.
|
@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
|
|
2
2
|
s.name = "request-log-analyzer"
|
3
3
|
|
4
4
|
# Do not set the version and date field manually, this is done by the release script
|
5
|
-
s.version = "1.12.
|
6
|
-
s.date = "2012-
|
5
|
+
s.version = "1.12.5"
|
6
|
+
s.date = "2012-09-20"
|
7
7
|
|
8
8
|
s.rubyforge_project = 'r-l-a'
|
9
9
|
|
@@ -45,6 +45,6 @@ Gem::Specification.new do |s|
|
|
45
45
|
|
46
46
|
# The files and test_files directives are set automatically by the release script.
|
47
47
|
# Do not change them by hand, but make sure to add the files to the git repository.
|
48
|
-
s.files = %w(.gitignore .infinity_test .travis.yml DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
|
48
|
+
s.files = %w(.gitignore .infinity_test .travis.yml DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/ordered_hash.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
|
49
49
|
s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
|
50
50
|
end
|
@@ -17,8 +17,6 @@ describe RequestLogAnalyzer::Database::Base do
|
|
17
17
|
@orm_class.stub!(:serialize)
|
18
18
|
@orm_class.stub!(:line_definition=)
|
19
19
|
|
20
|
-
Class.stub!(:new).with(RequestLogAnalyzer::Database::Base).and_return(@orm_class)
|
21
|
-
|
22
20
|
RequestLogAnalyzer::Database::Request.stub!(:has_many)
|
23
21
|
RequestLogAnalyzer::Database::Source.stub!(:has_many)
|
24
22
|
|
@@ -27,43 +25,42 @@ describe RequestLogAnalyzer::Database::Base do
|
|
27
25
|
end
|
28
26
|
|
29
27
|
it "should create a new subclass using the Base class as parent" do
|
30
|
-
|
31
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
28
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
32
29
|
end
|
33
30
|
|
34
31
|
it "should store the LineDefinition" do
|
35
32
|
@orm_class.should_receive(:line_definition=).with(@line_definition)
|
36
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
33
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
37
34
|
end
|
38
35
|
|
39
36
|
it "should set the table name for the subclass" do
|
40
37
|
@orm_class.should_receive("table_name=").with('test_lines')
|
41
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
38
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
42
39
|
end
|
43
40
|
|
44
41
|
it "should set the :belongs_to relationship with the Request class" do
|
45
42
|
@orm_class.should_receive(:belongs_to).with(:request, {:class_name=>"RequestLogAnalyzer::Database::Request"})
|
46
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
43
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
47
44
|
end
|
48
45
|
|
49
46
|
it "should set a :has_many relationship in the request class" do
|
50
47
|
RequestLogAnalyzer::Database::Request.should_receive(:has_many).with(:test_lines)
|
51
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
48
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
52
49
|
end
|
53
50
|
|
54
51
|
it "should set a :has_many relationship in the source class" do
|
55
52
|
RequestLogAnalyzer::Database::Source.should_receive(:has_many).with(:test_lines)
|
56
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
53
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
57
54
|
end
|
58
55
|
|
59
56
|
it "should set the :belongs_to relationship with the Source class" do
|
60
57
|
@orm_class.should_receive(:belongs_to).with(:source, {:class_name=>"RequestLogAnalyzer::Database::Source"})
|
61
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
58
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
62
59
|
end
|
63
60
|
|
64
61
|
it "should serialize a complex field" do
|
65
62
|
@orm_class.should_receive(:serialize).with(:evaluated, Hash)
|
66
|
-
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
|
63
|
+
RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition, @orm_class)
|
67
64
|
end
|
68
65
|
|
69
66
|
end
|
@@ -82,37 +79,31 @@ describe RequestLogAnalyzer::Database::Base do
|
|
82
79
|
@klass.stub!(:column_names).and_return(['id', 'request_id', 'source_id', 'lineno', 'duration'])
|
83
80
|
@klass.stub!("table_name=")
|
84
81
|
@klass.stub!(:belongs_to)
|
85
|
-
Class.stub!(:new).with(RequestLogAnalyzer::Database::Base).and_return(@klass)
|
86
|
-
end
|
87
|
-
|
88
|
-
it "should create a new subclass using the Base class as parent" do
|
89
|
-
Class.should_receive(:new).with(RequestLogAnalyzer::Database::Base).and_return(@klass)
|
90
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
91
82
|
end
|
92
83
|
|
93
84
|
it "should set the table name" do
|
94
85
|
@klass.should_receive("table_name=").with('completed_lines')
|
95
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
86
|
+
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines', @klass)
|
96
87
|
end
|
97
88
|
|
98
89
|
it "should create the :belongs_to relation to the request class" do
|
99
90
|
@klass.should_receive(:belongs_to).with(:request, {:class_name=>"RequestLogAnalyzer::Database::Request"})
|
100
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
91
|
+
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines', @klass)
|
101
92
|
end
|
102
93
|
|
103
94
|
it "should create the :has_many relation in the request class" do
|
104
95
|
RequestLogAnalyzer::Database::Request.should_receive(:has_many).with(:completed_lines)
|
105
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
96
|
+
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines', @klass)
|
106
97
|
end
|
107
98
|
|
108
99
|
it "should create the :belongs_to relation to the source class" do
|
109
100
|
@klass.should_receive(:belongs_to).with(:source, {:class_name=>"RequestLogAnalyzer::Database::Source"})
|
110
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
101
|
+
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines', @klass)
|
111
102
|
end
|
112
103
|
|
113
104
|
it "should create the :has_many relation in the request class" do
|
114
105
|
RequestLogAnalyzer::Database::Source.should_receive(:has_many).with(:completed_lines)
|
115
|
-
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
|
106
|
+
RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines', @klass)
|
116
107
|
end
|
117
108
|
|
118
109
|
end
|
metadata
CHANGED
@@ -1,80 +1,83 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: request-log-analyzer
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
|
5
|
-
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 1
|
7
|
+
- 12
|
8
|
+
- 5
|
9
|
+
version: 1.12.5
|
6
10
|
platform: ruby
|
7
|
-
authors:
|
11
|
+
authors:
|
8
12
|
- Willem van Bergen
|
9
13
|
- Bart ten Brinke
|
10
14
|
autorequire:
|
11
15
|
bindir: bin
|
12
16
|
cert_chain: []
|
13
|
-
|
14
|
-
|
15
|
-
-
|
17
|
+
|
18
|
+
date: 2012-09-20 00:00:00 -04:00
|
19
|
+
default_executable: request-log-analyzer
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
16
22
|
name: rake
|
17
|
-
requirement: &70130940523300 !ruby/object:Gem::Requirement
|
18
|
-
none: false
|
19
|
-
requirements:
|
20
|
-
- - ! '>='
|
21
|
-
- !ruby/object:Gem::Version
|
22
|
-
version: '0'
|
23
|
-
type: :development
|
24
23
|
prerelease: false
|
25
|
-
|
26
|
-
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
version: "0"
|
31
|
+
type: :development
|
32
|
+
version_requirements: *id001
|
33
|
+
- !ruby/object:Gem::Dependency
|
27
34
|
name: rspec
|
28
|
-
|
29
|
-
|
30
|
-
requirements:
|
35
|
+
prerelease: false
|
36
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
31
38
|
- - ~>
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
segments:
|
41
|
+
- 2
|
42
|
+
- 8
|
43
|
+
version: "2.8"
|
34
44
|
type: :development
|
35
|
-
|
36
|
-
|
37
|
-
- !ruby/object:Gem::Dependency
|
45
|
+
version_requirements: *id002
|
46
|
+
- !ruby/object:Gem::Dependency
|
38
47
|
name: activerecord
|
39
|
-
requirement: &70130940521320 !ruby/object:Gem::Requirement
|
40
|
-
none: false
|
41
|
-
requirements:
|
42
|
-
- - ! '>='
|
43
|
-
- !ruby/object:Gem::Version
|
44
|
-
version: '0'
|
45
|
-
type: :development
|
46
48
|
prerelease: false
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
-
version: '0'
|
49
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
segments:
|
54
|
+
- 0
|
55
|
+
version: "0"
|
56
56
|
type: :development
|
57
|
+
version_requirements: *id003
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: sqlite3
|
57
60
|
prerelease: false
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
email:
|
61
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
62
|
+
requirements:
|
63
|
+
- - ">="
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
segments:
|
66
|
+
- 0
|
67
|
+
version: "0"
|
68
|
+
type: :development
|
69
|
+
version_requirements: *id004
|
70
|
+
description: " Request log analyzer's purpose is to find out how your web application is being used, how it performs and to\n focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the \n information. Once it is finished parsing the log file(s), it will show the requests that take op most server time \n using various metrics. It can also insert all parsed request information into a database so you can roll your own\n analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL \n slow query logs out of the box, but file formats of other applications can easily be supported by supplying an \n easy to write log file format definition.\n"
|
71
|
+
email:
|
70
72
|
- willem@railsdoctors.com
|
71
73
|
- bart@railsdoctors.com
|
72
|
-
executables:
|
74
|
+
executables:
|
73
75
|
- request-log-analyzer
|
74
76
|
extensions: []
|
75
|
-
|
77
|
+
|
78
|
+
extra_rdoc_files:
|
76
79
|
- README.rdoc
|
77
|
-
files:
|
80
|
+
files:
|
78
81
|
- .gitignore
|
79
82
|
- .infinity_test
|
80
83
|
- .travis.yml
|
@@ -89,6 +92,7 @@ files:
|
|
89
92
|
- lib/cli/database_console_init.rb
|
90
93
|
- lib/cli/progressbar.rb
|
91
94
|
- lib/cli/tools.rb
|
95
|
+
- lib/ordered_hash.rb
|
92
96
|
- lib/request_log_analyzer.rb
|
93
97
|
- lib/request_log_analyzer/aggregator.rb
|
94
98
|
- lib/request_log_analyzer/aggregator/database_inserter.rb
|
@@ -217,40 +221,42 @@ files:
|
|
217
221
|
- spec/unit/tracker/traffic_tracker_spec.rb
|
218
222
|
- tasks/github-gem.rake
|
219
223
|
- tasks/request_log_analyzer.rake
|
224
|
+
has_rdoc: true
|
220
225
|
homepage: http://railsdoctors.com
|
221
226
|
licenses: []
|
227
|
+
|
222
228
|
post_install_message:
|
223
|
-
rdoc_options:
|
229
|
+
rdoc_options:
|
224
230
|
- --title
|
225
231
|
- request-log-analyzer
|
226
232
|
- --main
|
227
233
|
- README.rdoc
|
228
234
|
- --line-numbers
|
229
235
|
- --inline-source
|
230
|
-
require_paths:
|
236
|
+
require_paths:
|
231
237
|
- lib
|
232
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
requirements:
|
241
|
-
- -
|
242
|
-
- !ruby/object:Gem::Version
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
238
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
239
|
+
requirements:
|
240
|
+
- - ">="
|
241
|
+
- !ruby/object:Gem::Version
|
242
|
+
segments:
|
243
|
+
- 0
|
244
|
+
version: "0"
|
245
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
246
|
+
requirements:
|
247
|
+
- - ">="
|
248
|
+
- !ruby/object:Gem::Version
|
249
|
+
segments:
|
250
|
+
- 0
|
251
|
+
version: "0"
|
252
|
+
requirements:
|
253
|
+
- To use the database inserter, ActiveRecord and an appropriate database adapter are required.
|
247
254
|
rubyforge_project: r-l-a
|
248
|
-
rubygems_version: 1.
|
255
|
+
rubygems_version: 1.3.6
|
249
256
|
signing_key:
|
250
257
|
specification_version: 3
|
251
|
-
summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL
|
252
|
-
|
253
|
-
test_files:
|
258
|
+
summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers
|
259
|
+
test_files:
|
254
260
|
- spec/integration/command_line_usage_spec.rb
|
255
261
|
- spec/integration/mailer_spec.rb
|
256
262
|
- spec/integration/munin_plugins_rails_spec.rb
|