request-log-analyzer 1.3.7 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +3 -3
- data/README.rdoc +1 -1
- data/bin/request-log-analyzer +17 -14
- data/lib/cli/command_line_arguments.rb +51 -51
- data/lib/cli/database_console.rb +3 -3
- data/lib/cli/database_console_init.rb +2 -2
- data/lib/cli/progressbar.rb +10 -10
- data/lib/cli/tools.rb +3 -3
- data/lib/request_log_analyzer.rb +4 -4
- data/lib/request_log_analyzer/aggregator.rb +10 -10
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +9 -9
- data/lib/request_log_analyzer/aggregator/echo.rb +14 -9
- data/lib/request_log_analyzer/aggregator/summarizer.rb +26 -26
- data/lib/request_log_analyzer/controller.rb +153 -69
- data/lib/request_log_analyzer/database.rb +13 -13
- data/lib/request_log_analyzer/database/base.rb +17 -17
- data/lib/request_log_analyzer/database/connection.rb +3 -3
- data/lib/request_log_analyzer/database/request.rb +2 -2
- data/lib/request_log_analyzer/database/source.rb +1 -1
- data/lib/request_log_analyzer/file_format.rb +15 -15
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +16 -16
- data/lib/request_log_analyzer/file_format/apache.rb +20 -19
- data/lib/request_log_analyzer/file_format/merb.rb +12 -12
- data/lib/request_log_analyzer/file_format/rack.rb +4 -4
- data/lib/request_log_analyzer/file_format/rails.rb +146 -70
- data/lib/request_log_analyzer/file_format/rails_development.rb +4 -49
- data/lib/request_log_analyzer/filter.rb +6 -6
- data/lib/request_log_analyzer/filter/anonymize.rb +6 -6
- data/lib/request_log_analyzer/filter/field.rb +9 -9
- data/lib/request_log_analyzer/filter/timespan.rb +12 -10
- data/lib/request_log_analyzer/line_definition.rb +15 -14
- data/lib/request_log_analyzer/log_processor.rb +22 -22
- data/lib/request_log_analyzer/mailer.rb +15 -9
- data/lib/request_log_analyzer/output.rb +53 -12
- data/lib/request_log_analyzer/output/fixed_width.rb +40 -41
- data/lib/request_log_analyzer/output/html.rb +20 -20
- data/lib/request_log_analyzer/request.rb +35 -36
- data/lib/request_log_analyzer/source.rb +7 -7
- data/lib/request_log_analyzer/source/database_loader.rb +7 -7
- data/lib/request_log_analyzer/source/log_parser.rb +48 -43
- data/lib/request_log_analyzer/tracker.rb +128 -14
- data/lib/request_log_analyzer/tracker/duration.rb +39 -132
- data/lib/request_log_analyzer/tracker/frequency.rb +31 -32
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +20 -19
- data/lib/request_log_analyzer/tracker/timespan.rb +17 -17
- data/lib/request_log_analyzer/tracker/traffic.rb +36 -116
- data/request-log-analyzer.gemspec +19 -15
- data/spec/fixtures/rails_22.log +1 -1
- data/spec/integration/command_line_usage_spec.rb +1 -1
- data/spec/lib/helpers.rb +7 -7
- data/spec/lib/macros.rb +3 -3
- data/spec/lib/matchers.rb +41 -27
- data/spec/lib/mocks.rb +15 -14
- data/spec/lib/testing_format.rb +9 -9
- data/spec/spec_helper.rb +6 -6
- data/spec/unit/aggregator/database_inserter_spec.rb +13 -13
- data/spec/unit/aggregator/summarizer_spec.rb +4 -4
- data/spec/unit/controller/controller_spec.rb +2 -2
- data/spec/unit/controller/log_processor_spec.rb +1 -1
- data/spec/unit/database/base_class_spec.rb +19 -19
- data/spec/unit/database/connection_spec.rb +3 -3
- data/spec/unit/database/database_spec.rb +25 -25
- data/spec/unit/file_format/amazon_s3_format_spec.rb +5 -5
- data/spec/unit/file_format/apache_format_spec.rb +13 -13
- data/spec/unit/file_format/file_format_api_spec.rb +13 -13
- data/spec/unit/file_format/line_definition_spec.rb +24 -17
- data/spec/unit/file_format/merb_format_spec.rb +41 -45
- data/spec/unit/file_format/rails_format_spec.rb +157 -117
- data/spec/unit/filter/anonymize_filter_spec.rb +2 -2
- data/spec/unit/filter/field_filter_spec.rb +13 -13
- data/spec/unit/filter/filter_spec.rb +1 -1
- data/spec/unit/filter/timespan_filter_spec.rb +15 -15
- data/spec/unit/mailer_spec.rb +30 -0
- data/spec/unit/{source/request_spec.rb → request_spec.rb} +30 -30
- data/spec/unit/source/log_parser_spec.rb +27 -27
- data/spec/unit/tracker/duration_tracker_spec.rb +115 -78
- data/spec/unit/tracker/frequency_tracker_spec.rb +74 -63
- data/spec/unit/tracker/hourly_spread_spec.rb +28 -20
- data/spec/unit/tracker/timespan_tracker_spec.rb +25 -13
- data/spec/unit/tracker/tracker_api_spec.rb +13 -13
- data/spec/unit/tracker/traffic_tracker_spec.rb +81 -79
- data/tasks/github-gem.rake +125 -75
- data/tasks/request_log_analyzer.rake +2 -2
- metadata +8 -6
@@ -1,5 +1,5 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
2
|
+
|
3
3
|
# The Merb file format parses the request header with the timestamp, the params line
|
4
4
|
# with the most important request information and the durations line which contains
|
5
5
|
# the different request durations that can be used for analysis.
|
@@ -11,14 +11,14 @@ module RequestLogAnalyzer::FileFormat
|
|
11
11
|
line.teaser = /Started request handling\:/
|
12
12
|
line.regexp = /Started request handling\:\ (.+)/
|
13
13
|
line.captures << { :name => :timestamp, :type => :timestamp }
|
14
|
-
end
|
15
|
-
|
14
|
+
end
|
15
|
+
|
16
16
|
# ~ Params: {"action"=>"create", "controller"=>"session"}
|
17
17
|
# ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
|
18
18
|
line_definition :params do |line|
|
19
19
|
line.teaser = /Params\:\ /
|
20
20
|
line.regexp = /Params\:\ (\{.+\})/
|
21
|
-
line.captures << { :name => :params, :type => :eval, :provides => {
|
21
|
+
line.captures << { :name => :params, :type => :eval, :provides => {
|
22
22
|
:namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
|
23
23
|
end
|
24
24
|
|
@@ -31,32 +31,32 @@ module RequestLogAnalyzer::FileFormat
|
|
31
31
|
:dispatch_time => :duration, :after_filters_time => :duration,
|
32
32
|
:before_filters_time => :duration, :action_time => :duration } }
|
33
33
|
end
|
34
|
-
|
35
|
-
REQUEST_CATEGORIZER = Proc.new do |request|
|
34
|
+
|
35
|
+
REQUEST_CATEGORIZER = Proc.new do |request|
|
36
36
|
category = "#{request[:controller]}##{request[:action]}"
|
37
37
|
category = "#{request[:namespace]}::#{category}" if request[:namespace]
|
38
38
|
category = "#{category}.#{request[:format]}" if request[:format]
|
39
39
|
category
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
report do |analyze|
|
43
|
-
|
43
|
+
|
44
44
|
analyze.timespan
|
45
45
|
analyze.hourly_spread
|
46
46
|
|
47
|
-
analyze.frequency :category => REQUEST_CATEGORIZER, :
|
47
|
+
analyze.frequency :category => REQUEST_CATEGORIZER, :title => "Top 20 by hits"
|
48
48
|
analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
|
49
|
-
|
49
|
+
|
50
50
|
# analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
|
51
51
|
# analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
|
52
52
|
# analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
|
53
53
|
end
|
54
54
|
|
55
55
|
class Request < RequestLogAnalyzer::Request
|
56
|
-
|
56
|
+
|
57
57
|
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
58
58
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
59
|
-
|
59
|
+
|
60
60
|
# Speed up timestamp conversion
|
61
61
|
def convert_timestamp(value, definition)
|
62
62
|
"#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
|
@@ -1,98 +1,174 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
2
|
+
|
3
|
+
# Default FileFormat class for Rails logs.
|
4
|
+
#
|
5
|
+
# Instances will be created dynamically based on the lines you want it to parse. You can
|
6
|
+
# specify what lines should be included in the parser by providing a list to the create
|
7
|
+
# method as first argument.
|
3
8
|
class Rails < Base
|
4
|
-
|
5
|
-
# Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
|
6
|
-
line_definition :processing do |line|
|
7
|
-
line.header = true # this line is the first log line for a request
|
8
|
-
line.teaser = /Processing /
|
9
|
-
line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
|
10
|
-
line.captures << { :name => :controller, :type => :string } \
|
11
|
-
<< { :name => :action, :type => :string } \
|
12
|
-
<< { :name => :format, :type => :string, :default => 'html' } \
|
13
|
-
<< { :name => :ip, :type => :string } \
|
14
|
-
<< { :name => :timestamp, :type => :timestamp } \
|
15
|
-
<< { :name => :method, :type => :string }
|
16
|
-
end
|
17
9
|
|
18
|
-
#
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
#
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
10
|
+
# Creates a Rails FileFormat instance.
|
11
|
+
#
|
12
|
+
# The lines that will be parsed can be defined by the argument to this function,
|
13
|
+
# which should be an array of line names, or a list of line names as comma separated
|
14
|
+
# string. The resulting report depends on the lines that will be parsed. You can
|
15
|
+
# also provide s string that describes a common set of lines, like "production",
|
16
|
+
# "development" or "production".
|
17
|
+
def self.create(lines = 'production')
|
18
|
+
definitions_hash = line_definer.line_definitions.clone
|
19
|
+
|
20
|
+
lines = lines.to_s.split(',') if lines.kind_of?(String)
|
21
|
+
lines = [lines.to_s] if lines.kind_of?(Symbol)
|
22
|
+
|
23
|
+
lines.each do |line|
|
24
|
+
line = line.to_sym
|
25
|
+
if LINE_COLLECTIONS.has_key?(line)
|
26
|
+
LINE_COLLECTIONS[line].each { |l| definitions_hash[l] = LINE_DEFINITIONS[l] }
|
27
|
+
elsif LINE_DEFINITIONS.has_key?(line)
|
28
|
+
definitions_hash[line] = LINE_DEFINITIONS[line]
|
29
|
+
else
|
30
|
+
raise "Unrecognized Rails log line name: #{line.inspect}!"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
return self.new(definitions_hash, report_trackers(definitions_hash))
|
32
35
|
end
|
36
|
+
|
37
|
+
# Creates trackers based on the specified line definitions.
|
38
|
+
#
|
39
|
+
# The more lines that will be parsed, the more information will appear in the report.
|
40
|
+
def self.report_trackers(lines)
|
41
|
+
analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
|
42
|
+
|
43
|
+
analyze.timespan
|
44
|
+
analyze.hourly_spread
|
45
|
+
|
46
|
+
analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Most requested'
|
47
|
+
analyze.frequency :method, :title => 'HTTP methods'
|
48
|
+
analyze.frequency :status, :title => 'HTTP statuses returned'
|
49
|
+
|
50
|
+
if lines.has_key?(:cache_hit)
|
51
|
+
analyze.frequency(:category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' },
|
52
|
+
:title => 'Rails action cache hits')
|
53
|
+
end
|
54
|
+
|
55
|
+
analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration", :line_type => :completed
|
56
|
+
analyze.duration :view, :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
|
57
|
+
analyze.duration :db, :category => REQUEST_CATEGORIZER, :title => "Database time", :line_type => :completed
|
58
|
+
|
59
|
+
analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
|
60
|
+
:if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
61
|
+
|
62
|
+
if lines.has_key?(:failure)
|
63
|
+
analyze.frequency :error, :title => 'Failed requests', :line_type => :failure
|
64
|
+
end
|
33
65
|
|
66
|
+
if lines.has_key?(:rendered)
|
67
|
+
analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true, :title => 'Partial rendering duration'
|
68
|
+
end
|
69
|
+
|
70
|
+
if lines.has_key?(:query_executed)
|
71
|
+
analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true, :title => 'Query duration'
|
72
|
+
end
|
73
|
+
|
74
|
+
return analyze.trackers + report_definer.trackers
|
75
|
+
end
|
34
76
|
|
35
77
|
# Rails < 2.1 completed line example
|
36
78
|
# Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
|
37
79
|
RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
|
38
80
|
|
39
81
|
# Rails > 2.1 completed line example
|
40
|
-
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
82
|
+
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
41
83
|
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
|
42
84
|
|
43
|
-
#
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
85
|
+
# A hash of definitions for all common lines in Rails logs.
|
86
|
+
LINE_DEFINITIONS = {
|
87
|
+
:processing => RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,
|
88
|
+
:teaser => /Processing /,
|
89
|
+
:regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
|
90
|
+
:captures => [{ :name => :controller, :type => :string },
|
91
|
+
{ :name => :action, :type => :string },
|
92
|
+
{ :name => :format, :type => :string, :default => 'html' },
|
93
|
+
{ :name => :ip, :type => :string },
|
94
|
+
{ :name => :timestamp, :type => :timestamp },
|
95
|
+
{ :name => :method, :type => :string }]),
|
96
|
+
|
97
|
+
:completed => RequestLogAnalyzer::LineDefinition.new(:completed, :footer => true,
|
98
|
+
:teaser => /Completed in /,
|
99
|
+
:regexp => Regexp.union(RAILS_21_COMPLETED, RAILS_22_COMPLETED),
|
100
|
+
:captures => [{ :name => :duration, :type => :duration, :unit => :sec }, # First old variant capture
|
101
|
+
{ :name => :view, :type => :duration, :unit => :sec },
|
102
|
+
{ :name => :db, :type => :duration, :unit => :sec },
|
103
|
+
{ :name => :status, :type => :integer },
|
104
|
+
{ :name => :url, :type => :string }, # Last old variant capture
|
105
|
+
{ :name => :duration, :type => :duration, :unit => :msec }, # First new variant capture
|
106
|
+
{ :name => :view, :type => :duration, :unit => :msec },
|
107
|
+
{ :name => :db, :type => :duration, :unit => :msec },
|
108
|
+
{ :name => :status, :type => :integer },
|
109
|
+
{ :name => :url, :type => :string }]), # Last new variant capture
|
64
110
|
|
111
|
+
:failure => RequestLogAnalyzer::LineDefinition.new(:failure, :footer => true,
|
112
|
+
:teaser => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:/,
|
113
|
+
:regexp => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/,
|
114
|
+
:captures => [{ :name => :error, :type => :string },
|
115
|
+
{ :name => :message, :type => :string },
|
116
|
+
{ :name => :line, :type => :integer },
|
117
|
+
{ :name => :file, :type => :string }]),
|
118
|
+
|
119
|
+
:cache_hit => RequestLogAnalyzer::LineDefinition.new(:cache_hit,
|
120
|
+
:regexp => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter/),
|
121
|
+
|
122
|
+
:parameters => RequestLogAnalyzer::LineDefinition.new(:parameters,
|
123
|
+
:teaser => / Parameters:/,
|
124
|
+
:regexp => / Parameters:\s+(\{.*\})/,
|
125
|
+
:captures => [{ :name => :params, :type => :eval }]),
|
126
|
+
|
127
|
+
:rendered => RequestLogAnalyzer::LineDefinition.new(:rendered,
|
128
|
+
:teaser => /Rendered /,
|
129
|
+
:regexp => /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
|
130
|
+
:captures => [{ :name => :render_file, :type => :string },
|
131
|
+
{ :name => :render_duration, :type => :duration, :unit => :msec }]),
|
132
|
+
|
133
|
+
:query_executed => RequestLogAnalyzer::LineDefinition.new(:query_executed,
|
134
|
+
:regexp => /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/,
|
135
|
+
:captures => [{ :name => :query_class, :type => :string },
|
136
|
+
{ :name => :query_duration, :type => :duration, :unit => :msec },
|
137
|
+
{ :name => :query_sql, :type => :sql }]),
|
138
|
+
|
139
|
+
:query_cached => RequestLogAnalyzer::LineDefinition.new(:query_cached,
|
140
|
+
:regexp => /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/,
|
141
|
+
:captures => [{ :name => :cached_duration, :type => :duration, :unit => :msec },
|
142
|
+
{ :name => :cached_sql, :type => :sql }])
|
143
|
+
}
|
144
|
+
|
145
|
+
# Definitions of common combinations of lines that can be parsed
|
146
|
+
LINE_COLLECTIONS = {
|
147
|
+
:minimal => [:processing, :completed],
|
148
|
+
:production => [:processing, :completed, :failure, :cache_hit],
|
149
|
+
:development => [:processing, :completed, :failure, :rendered, :query_executed, :query_cached],
|
150
|
+
:all => LINE_DEFINITIONS.keys
|
151
|
+
}
|
152
|
+
|
153
|
+
|
154
|
+
# Simple function to categorize Rails requests using controller/actions/format and method.
|
65
155
|
REQUEST_CATEGORIZER = Proc.new do |request|
|
66
156
|
"#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
|
67
157
|
end
|
68
158
|
|
69
|
-
report do |analyze|
|
70
|
-
analyze.timespan
|
71
|
-
analyze.hourly_spread
|
72
|
-
|
73
|
-
analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20
|
74
|
-
analyze.frequency :method, :title => 'HTTP methods'
|
75
|
-
analyze.frequency :status, :title => 'HTTP statuses returned'
|
76
|
-
analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
|
77
|
-
|
78
|
-
analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration", :line_type => :completed
|
79
|
-
analyze.duration :view, :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
|
80
|
-
analyze.duration :db, :category => REQUEST_CATEGORIZER, :title => "Database time", :line_type => :completed
|
81
|
-
|
82
|
-
analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
|
83
|
-
:if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
|
84
|
-
|
85
|
-
analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
|
86
|
-
end
|
87
|
-
|
88
159
|
# Define a custom Request class for the Rails file format to speed up timestamp handling
|
89
160
|
# and to ensure that a format is always set.
|
90
161
|
class Request < RequestLogAnalyzer::Request
|
91
|
-
|
162
|
+
|
92
163
|
# Do not use DateTime.parse
|
93
164
|
def convert_timestamp(value, definition)
|
94
165
|
value.gsub(/[^0-9]/, '')[0...14].to_i
|
95
166
|
end
|
167
|
+
|
168
|
+
# Sanitizes SQL queries so that they can be grouped
|
169
|
+
def convert_sql(sql, definition)
|
170
|
+
sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
|
171
|
+
end
|
96
172
|
end
|
97
173
|
|
98
174
|
end
|
@@ -1,57 +1,12 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
2
|
+
|
3
3
|
# The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes
|
4
4
|
# all lines of the normal Rails file format, but parses SQL queries and partial rendering lines
|
5
5
|
# as well.
|
6
6
|
class RailsDevelopment < Rails
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
line.teaser = /Parameters/
|
11
|
-
line.regexp = /\s+Parameters:\s+(\{.*\})/
|
12
|
-
line.captures << { :name => :params, :type => :eval }
|
13
|
-
end
|
14
|
-
|
15
|
-
# Rendered layouts/_footer (2.9ms)
|
16
|
-
line_definition :rendered do |line|
|
17
|
-
line.regexp = /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/
|
18
|
-
line.captures << { :name => :render_file, :type => :string } \
|
19
|
-
<< { :name => :render_duration, :type => :duration, :unit => :msec }
|
20
|
-
end
|
21
|
-
|
22
|
-
# [4;36;1mUser Load (0.4ms)[0m [0;1mSELECT * FROM `users` WHERE (`users`.`id` = 18205844) [0m
|
23
|
-
line_definition :query_executed do |line|
|
24
|
-
line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/
|
25
|
-
line.captures << { :name => :query_class, :type => :string } \
|
26
|
-
<< { :name => :query_duration, :type => :duration, :unit => :msec } \
|
27
|
-
<< { :name => :query_sql, :type => :sql }
|
28
|
-
end
|
29
|
-
|
30
|
-
# [4;35;1mCACHE (0.0ms)[0m [0mSELECT * FROM `users` WHERE (`users`.`id` = 0) [0m
|
31
|
-
line_definition :query_cached do |line|
|
32
|
-
line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/
|
33
|
-
line.captures << { :name => :cached_duration, :type => :duration, :unit => :msec } \
|
34
|
-
<< { :name => :cached_sql, :type => :sql }
|
35
|
-
end
|
36
|
-
|
37
|
-
# Define the reporting for the additional parsed lines
|
38
|
-
report(:append) do |analyze|
|
39
|
-
|
40
|
-
analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true,
|
41
|
-
:amount => 20, :title => 'Partial rendering duration'
|
42
|
-
|
43
|
-
analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true,
|
44
|
-
:amount => 20, :title => 'Query duration'
|
45
|
-
|
46
|
-
end
|
47
|
-
|
48
|
-
# Add a converter method for the SQL fields the the Rails request class
|
49
|
-
class Request < Rails::Request
|
50
|
-
|
51
|
-
# Sanitizes SQL queries so that they can be grouped
|
52
|
-
def convert_sql(sql, definition)
|
53
|
-
sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
|
54
|
-
end
|
7
|
+
def self.create
|
8
|
+
puts 'DEPRECATED: use --rails-format development instead!'
|
9
|
+
super('development')
|
55
10
|
end
|
56
11
|
end
|
57
12
|
end
|
@@ -1,17 +1,17 @@
|
|
1
1
|
module RequestLogAnalyzer::Filter
|
2
|
-
|
2
|
+
|
3
3
|
# Filter class loader using const_missing
|
4
4
|
# This function will automatically load the class file based on the name of the class
|
5
5
|
def self.const_missing(const)
|
6
6
|
RequestLogAnalyzer::load_default_class_file(self, const)
|
7
7
|
end
|
8
|
-
|
8
|
+
|
9
9
|
# Base filter class used to filter input requests.
|
10
10
|
# All filters should interit from this base.
|
11
11
|
class Base
|
12
|
-
|
12
|
+
|
13
13
|
attr_reader :file_format, :options
|
14
|
-
|
14
|
+
|
15
15
|
# Initializer
|
16
16
|
# <tt>format</tt> The file format
|
17
17
|
# <tt>options</tt> Are passed to the filters.
|
@@ -19,12 +19,12 @@ module RequestLogAnalyzer::Filter
|
|
19
19
|
@file_format = format
|
20
20
|
@options = options
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
# Return the request if the request should be kept.
|
24
24
|
# Return nil otherwise.
|
25
25
|
def filter(request)
|
26
26
|
request
|
27
27
|
end
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
module RequestLogAnalyzer::Filter
|
2
|
-
|
2
|
+
|
3
3
|
# Filter to anonymize parsed values
|
4
4
|
# Options
|
5
5
|
# * <tt>:mode</tt> :reject or :accept.
|
6
6
|
# * <tt>:field</tt> Specific field to accept or reject.
|
7
7
|
# * <tt>:value</tt> Value that the field should match to be accepted or rejected.
|
8
8
|
class Anonymize < Base
|
9
|
-
|
9
|
+
|
10
10
|
def generate_random_ip
|
11
11
|
"#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
|
12
12
|
end
|
@@ -14,7 +14,7 @@ module RequestLogAnalyzer::Filter
|
|
14
14
|
def anonymize_url(value)
|
15
15
|
return value.sub(/^https?\:\/\/[A-Za-z0-9\.-]+\//, "http://example.com/")
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
def fuzz(value)
|
19
19
|
value * ((75 + rand(50)) / 100.0)
|
20
20
|
end
|
@@ -31,9 +31,9 @@ module RequestLogAnalyzer::Filter
|
|
31
31
|
request.attributes[key] = fuzz(value)
|
32
32
|
end
|
33
33
|
end
|
34
|
-
|
34
|
+
|
35
35
|
return request
|
36
|
-
end
|
36
|
+
end
|
37
37
|
end
|
38
|
-
|
38
|
+
|
39
39
|
end
|