request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -1,20 +1,18 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# PostgresQL spec 8.3.7
|
4
3
|
class Postgresql < Base
|
5
|
-
|
6
4
|
extend CommonRegularExpressions
|
7
|
-
|
5
|
+
|
8
6
|
line_definition :query do |line|
|
9
7
|
line.header = true
|
10
8
|
line.teaser = /.*LOG\:/
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%d %k:%M:%S')})\ \S+ \[\d+\]\:\ \[.*\]\ LOG\:\ \ \d+\:\ duration\: (.*)\ ms\ \ statement:\ (.*)/
|
12
10
|
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
14
|
-
line.capture(:query_time).as(:duration, :
|
12
|
+
line.capture(:query_time).as(:duration, unit: :sec)
|
15
13
|
line.capture(:query_fragment)
|
16
14
|
end
|
17
|
-
|
15
|
+
|
18
16
|
line_definition :location do |line|
|
19
17
|
line.footer = true
|
20
18
|
line.teaser = /.*LOCATION:/
|
@@ -22,7 +20,7 @@ module RequestLogAnalyzer::FileFormat
|
|
22
20
|
|
23
21
|
line.capture(:query).as(:sql) # Hack to gather up fragments
|
24
22
|
end
|
25
|
-
|
23
|
+
|
26
24
|
line_definition :query_fragment do |line|
|
27
25
|
line.regexp = /^(?!.*LOG)\s*(.*)\s*/
|
28
26
|
line.capture(:query_fragment)
|
@@ -30,14 +28,12 @@ module RequestLogAnalyzer::FileFormat
|
|
30
28
|
|
31
29
|
report do |analyze|
|
32
30
|
analyze.timespan
|
33
|
-
analyze.hourly_spread
|
34
|
-
analyze.duration :query_time, :
|
31
|
+
analyze.hourly_spread
|
32
|
+
analyze.duration :query_time, category: :query, title: 'Query time'
|
35
33
|
end
|
36
|
-
|
37
|
-
class Request < RequestLogAnalyzer::Request
|
38
34
|
|
39
|
-
|
40
|
-
|
35
|
+
class Request < RequestLogAnalyzer::Request
|
36
|
+
def convert_sql(value, _definition)
|
41
37
|
# Recreate the full SQL query by joining all the previous parts and this last line
|
42
38
|
sql = every(:query_fragment).join("\n") + value
|
43
39
|
|
@@ -52,7 +48,7 @@ module RequestLogAnalyzer::FileFormat
|
|
52
48
|
sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list
|
53
49
|
sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list
|
54
50
|
|
55
|
-
|
51
|
+
sql.lstrip.rstrip
|
56
52
|
end
|
57
53
|
|
58
54
|
def host
|
@@ -60,9 +56,9 @@ module RequestLogAnalyzer::FileFormat
|
|
60
56
|
end
|
61
57
|
|
62
58
|
# Convert the timestamp to an integer
|
63
|
-
def convert_timestamp(value,
|
59
|
+
def convert_timestamp(value, _definition)
|
64
60
|
_, y, m, d, h, i, s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
65
|
-
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
61
|
+
('20%s%s%s%s%s%s' % [y, m, d, h.rjust(2, '0'), i, s]).to_i
|
66
62
|
end
|
67
63
|
end
|
68
64
|
end
|
@@ -1,79 +1,77 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# Default FileFormat class for Rails logs.
|
4
3
|
#
|
5
4
|
# Instances will be created dynamically based on the lines you want it to parse. You can
|
6
5
|
# specify what lines should be included in the parser by providing a list to the create
|
7
6
|
# method as first argument.
|
8
7
|
class Rails < Base
|
9
|
-
|
10
8
|
extend CommonRegularExpressions
|
11
9
|
|
12
10
|
# Creates a Rails FileFormat instance.
|
13
11
|
#
|
14
|
-
# The lines that will be parsed can be defined by the argument to this function,
|
12
|
+
# The lines that will be parsed can be defined by the argument to this function,
|
15
13
|
# which should be an array of line names, or a list of line names as comma separated
|
16
|
-
# string. The resulting report depends on the lines that will be parsed. You can
|
14
|
+
# string. The resulting report depends on the lines that will be parsed. You can
|
17
15
|
# also provide s string that describes a common set of lines, like "production",
|
18
16
|
# "development" or "production".
|
19
17
|
def self.create(lines = 'production')
|
20
18
|
definitions_hash = line_definer.line_definitions.clone
|
21
|
-
|
22
|
-
lines = lines.to_s.split(',') if lines.
|
23
|
-
lines = [lines.to_s] if lines.
|
24
|
-
|
19
|
+
|
20
|
+
lines = lines.to_s.split(',') if lines.is_a?(String)
|
21
|
+
lines = [lines.to_s] if lines.is_a?(Symbol)
|
22
|
+
|
25
23
|
lines.each do |line|
|
26
24
|
line = line.to_sym
|
27
|
-
if LINE_COLLECTIONS.
|
25
|
+
if LINE_COLLECTIONS.key?(line)
|
28
26
|
LINE_COLLECTIONS[line].each { |l| definitions_hash[l] ||= LINE_DEFINITIONS[l] }
|
29
|
-
elsif LINE_DEFINITIONS.
|
27
|
+
elsif LINE_DEFINITIONS.key?(line)
|
30
28
|
definitions_hash[line] ||= LINE_DEFINITIONS[line]
|
31
29
|
else
|
32
|
-
|
30
|
+
fail "Unrecognized Rails log line name: #{line.inspect}!"
|
33
31
|
end
|
34
32
|
end
|
35
33
|
|
36
|
-
|
34
|
+
new(definitions_hash, report_trackers(definitions_hash))
|
37
35
|
end
|
38
|
-
|
36
|
+
|
39
37
|
# Creates trackers based on the specified line definitions.
|
40
38
|
#
|
41
39
|
# The more lines that will be parsed, the more information will appear in the report.
|
42
40
|
def self.report_trackers(lines)
|
43
41
|
analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
|
44
|
-
|
42
|
+
|
45
43
|
analyze.timespan
|
46
44
|
analyze.hourly_spread
|
47
|
-
|
48
|
-
analyze.frequency :
|
49
|
-
analyze.frequency :method, :
|
50
|
-
analyze.frequency :status, :
|
51
|
-
|
52
|
-
if lines.
|
53
|
-
analyze.frequency(:
|
54
|
-
|
45
|
+
|
46
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Most requested'
|
47
|
+
analyze.frequency :method, title: 'HTTP methods'
|
48
|
+
analyze.frequency :status, title: 'HTTP statuses returned'
|
49
|
+
|
50
|
+
if lines.key?(:cache_hit)
|
51
|
+
analyze.frequency(category: lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' },
|
52
|
+
title: 'Rails action cache hits')
|
55
53
|
end
|
56
|
-
|
57
|
-
analyze.duration :duration, :
|
58
|
-
analyze.duration :view, :
|
59
|
-
analyze.duration :db, :
|
60
|
-
|
61
|
-
analyze.frequency :
|
62
|
-
:
|
63
|
-
|
64
|
-
if lines.
|
65
|
-
analyze.frequency :error, :
|
54
|
+
|
55
|
+
analyze.duration :duration, category: REQUEST_CATEGORIZER, title: 'Request duration', line_type: :completed
|
56
|
+
analyze.duration :view, category: REQUEST_CATEGORIZER, title: 'View rendering time', line_type: :completed
|
57
|
+
analyze.duration :db, category: REQUEST_CATEGORIZER, title: 'Database time', line_type: :completed
|
58
|
+
|
59
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Process blockers (> 1 sec duration)',
|
60
|
+
if: lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
61
|
+
|
62
|
+
if lines.key?(:failure)
|
63
|
+
analyze.frequency :error, title: 'Failed requests', line_type: :failure
|
66
64
|
end
|
67
65
|
|
68
|
-
if lines.
|
69
|
-
analyze.duration :render_duration, :
|
66
|
+
if lines.key?(:rendered)
|
67
|
+
analyze.duration :render_duration, category: :render_file, multiple: true, title: 'Partial rendering duration'
|
70
68
|
end
|
71
69
|
|
72
|
-
if lines.
|
73
|
-
analyze.duration :query_duration, :
|
70
|
+
if lines.key?(:query_executed)
|
71
|
+
analyze.duration :query_duration, category: :query_sql, multiple: true, title: 'Query duration'
|
74
72
|
end
|
75
|
-
|
76
|
-
|
73
|
+
|
74
|
+
analyze.trackers + report_definer.trackers
|
77
75
|
end
|
78
76
|
|
79
77
|
# Rails < 2.1 completed line example
|
@@ -83,92 +81,90 @@ module RequestLogAnalyzer::FileFormat
|
|
83
81
|
# Rails > 2.1 completed line example
|
84
82
|
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
85
83
|
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+))?,?(?:.?DB: (\d+))?\)? \| (\d{3}).+\[(http.+)\]/
|
86
|
-
|
84
|
+
|
87
85
|
# A hash of definitions for all common lines in Rails logs.
|
88
86
|
LINE_DEFINITIONS = {
|
89
|
-
:
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
:
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
:
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
:
|
122
|
-
|
123
|
-
|
124
|
-
:
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
:
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
:
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
:
|
142
|
-
|
143
|
-
|
144
|
-
|
87
|
+
processing: RequestLogAnalyzer::LineDefinition.new(:processing, header: true,
|
88
|
+
teaser: /Processing /,
|
89
|
+
regexp: /Processing ((?:\w+::)*\w+)#(\w+)(?: to (\w+))? \(for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})\) \[([A-Z]+)\]/,
|
90
|
+
captures: [{ name: :controller, type: :string },
|
91
|
+
{ name: :action, type: :string },
|
92
|
+
{ name: :format, type: :string, default: 'html' },
|
93
|
+
{ name: :ip, type: :string },
|
94
|
+
{ name: :timestamp, type: :timestamp },
|
95
|
+
{ name: :method, type: :string }]),
|
96
|
+
|
97
|
+
completed: RequestLogAnalyzer::LineDefinition.new(:completed, footer: true,
|
98
|
+
teaser: /Completed in /,
|
99
|
+
regexp: Regexp.union(RAILS_21_COMPLETED, RAILS_22_COMPLETED),
|
100
|
+
captures: [{ name: :duration, type: :duration, unit: :sec }, # First old variant capture
|
101
|
+
{ name: :view, type: :duration, unit: :sec },
|
102
|
+
{ name: :db, type: :duration, unit: :sec },
|
103
|
+
{ name: :status, type: :integer },
|
104
|
+
{ name: :url, type: :string }, # Last old variant capture
|
105
|
+
{ name: :duration, type: :duration, unit: :msec }, # First new variant capture
|
106
|
+
{ name: :view, type: :duration, unit: :msec },
|
107
|
+
{ name: :db, type: :duration, unit: :msec },
|
108
|
+
{ name: :status, type: :integer },
|
109
|
+
{ name: :url, type: :string }]), # Last new variant capture
|
110
|
+
|
111
|
+
failure: RequestLogAnalyzer::LineDefinition.new(:failure, footer: true,
|
112
|
+
teaser: /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:/,
|
113
|
+
regexp: /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/,
|
114
|
+
captures: [{ name: :error, type: :string },
|
115
|
+
{ name: :message, type: :string },
|
116
|
+
{ name: :line, type: :integer },
|
117
|
+
{ name: :file, type: :string }]),
|
118
|
+
|
119
|
+
cache_hit: RequestLogAnalyzer::LineDefinition.new(:cache_hit,
|
120
|
+
regexp: /Filter chain halted as \[\#<ActionController::Filters::AroundFilter.*\@method=.*(?:Caching::Actions::ActionCacheFilter|action_controller\/caching\/actions\.rb).*\] did_not_yield/),
|
121
|
+
|
122
|
+
parameters: RequestLogAnalyzer::LineDefinition.new(:parameters,
|
123
|
+
teaser: / Parameters:/,
|
124
|
+
regexp: / Parameters:\s+(\{.*\})/,
|
125
|
+
captures: [{ name: :params, type: :eval }]),
|
126
|
+
|
127
|
+
rendered: RequestLogAnalyzer::LineDefinition.new(:rendered,
|
128
|
+
teaser: /Rendered /,
|
129
|
+
regexp: /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
|
130
|
+
captures: [{ name: :render_file, type: :string },
|
131
|
+
{ name: :render_duration, type: :duration, unit: :msec }]),
|
132
|
+
|
133
|
+
query_executed: RequestLogAnalyzer::LineDefinition.new(:query_executed,
|
134
|
+
regexp: /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/,
|
135
|
+
captures: [{ name: :query_class, type: :string },
|
136
|
+
{ name: :query_duration, type: :duration, unit: :msec },
|
137
|
+
{ name: :query_sql, type: :sql }]),
|
138
|
+
|
139
|
+
query_cached: RequestLogAnalyzer::LineDefinition.new(:query_cached,
|
140
|
+
regexp: /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/,
|
141
|
+
captures: [{ name: :cached_duration, type: :duration, unit: :msec },
|
142
|
+
{ name: :cached_sql, type: :sql }])
|
145
143
|
}
|
146
144
|
|
147
145
|
# Definitions of common combinations of lines that can be parsed
|
148
146
|
LINE_COLLECTIONS = {
|
149
|
-
:
|
150
|
-
:
|
151
|
-
:
|
152
|
-
:
|
147
|
+
minimal: [:processing, :completed],
|
148
|
+
production: [:processing, :completed, :failure, :cache_hit],
|
149
|
+
development: [:processing, :completed, :failure, :rendered, :query_executed, :query_cached],
|
150
|
+
all: LINE_DEFINITIONS.keys
|
153
151
|
}
|
154
152
|
|
155
|
-
|
156
153
|
# Simple function to categorize Rails requests using controller/actions/format and method.
|
157
|
-
REQUEST_CATEGORIZER =
|
154
|
+
REQUEST_CATEGORIZER = proc do |request|
|
158
155
|
"#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
|
159
156
|
end
|
160
157
|
|
161
158
|
# Define a custom Request class for the Rails file format to speed up timestamp handling
|
162
159
|
# and to ensure that a format is always set.
|
163
160
|
class Request < RequestLogAnalyzer::Request
|
164
|
-
|
165
161
|
# Do not use DateTime.parse
|
166
|
-
def convert_timestamp(value,
|
162
|
+
def convert_timestamp(value, _definition)
|
167
163
|
value.gsub(/[^0-9]/, '')[0...14].to_i
|
168
164
|
end
|
169
165
|
|
170
166
|
# Sanitizes SQL queries so that they can be grouped
|
171
|
-
def convert_sql(sql,
|
167
|
+
def convert_sql(sql, _definition)
|
172
168
|
sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
|
173
169
|
end
|
174
170
|
end
|
@@ -1,11 +1,9 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# Default FileFormat class for Rails 3 logs.
|
4
3
|
#
|
5
4
|
# For now, this is just a basic implementation. It will probaby change after
|
6
5
|
# Rails 3 final has been released.
|
7
6
|
class Rails3 < Base
|
8
|
-
|
9
7
|
extend CommonRegularExpressions
|
10
8
|
|
11
9
|
# beta4: Started GET "/" for 127.0.0.1 at Wed Jul 07 09:13:27 -0700 2010 (different time format)
|
@@ -46,9 +44,9 @@ module RequestLogAnalyzer::FileFormat
|
|
46
44
|
line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
|
47
45
|
|
48
46
|
line.capture(:status).as(:integer)
|
49
|
-
line.capture(:duration).as(:duration, :
|
50
|
-
line.capture(:view).as(:duration, :
|
51
|
-
line.capture(:db).as(:duration, :
|
47
|
+
line.capture(:duration).as(:duration, unit: :msec)
|
48
|
+
line.capture(:view).as(:duration, unit: :msec)
|
49
|
+
line.capture(:db).as(:duration, unit: :msec)
|
52
50
|
end
|
53
51
|
|
54
52
|
# ActionController::RoutingError (No route matches [GET] "/missing_stuff"):
|
@@ -76,7 +74,7 @@ module RequestLogAnalyzer::FileFormat
|
|
76
74
|
line.teaser = /\bRendered /
|
77
75
|
line.regexp = /\bRendered ([a-zA-Z0-9_\-\/.]+(?:\/[a-zA-Z0-9_\-.]+)+)(?:\ within\ .*?)? \((\d+(?:\.\d+)?)ms\)/
|
78
76
|
line.capture(:rendered_file)
|
79
|
-
line.capture(:partial_duration).as(:duration, :
|
77
|
+
line.capture(:partial_duration).as(:duration, unit: :msec)
|
80
78
|
end
|
81
79
|
|
82
80
|
# # Not parsed at the moment:
|
@@ -91,46 +89,44 @@ module RequestLogAnalyzer::FileFormat
|
|
91
89
|
analyze.timespan
|
92
90
|
analyze.hourly_spread
|
93
91
|
|
94
|
-
analyze.frequency :
|
95
|
-
analyze.frequency :method, :
|
96
|
-
analyze.frequency :status, :
|
92
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Most requested'
|
93
|
+
analyze.frequency :method, title: 'HTTP methods'
|
94
|
+
analyze.frequency :status, title: 'HTTP statuses returned'
|
97
95
|
|
98
|
-
analyze.duration :duration, :
|
99
|
-
analyze.duration :partial_duration, :
|
100
|
-
analyze.duration :view, :
|
101
|
-
analyze.duration :db, :
|
96
|
+
analyze.duration :duration, category: REQUEST_CATEGORIZER, title: 'Request duration', line_type: :completed
|
97
|
+
analyze.duration :partial_duration, category: :rendered_file, title: 'Partials rendering time', line_type: :rendered
|
98
|
+
analyze.duration :view, category: REQUEST_CATEGORIZER, title: 'View rendering time', line_type: :completed
|
99
|
+
analyze.duration :db, category: REQUEST_CATEGORIZER, title: 'Database time', line_type: :completed
|
102
100
|
|
103
|
-
analyze.frequency :
|
104
|
-
:
|
101
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Process blockers (> 1 sec duration)',
|
102
|
+
if: lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
105
103
|
|
106
|
-
analyze.frequency :
|
107
|
-
:
|
104
|
+
analyze.frequency category: lambda { |x| "[#{x[:missing_resource_method]}] #{x[:missing_resource]}" },
|
105
|
+
title: 'Routing Errors', if: lambda { |request| !request[:missing_resource].nil? }
|
108
106
|
end
|
109
107
|
|
110
108
|
class Request < RequestLogAnalyzer::Request
|
111
109
|
# Used to handle conversion of abbrev. month name to a digit
|
112
110
|
MONTHS = %w(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
|
113
111
|
|
114
|
-
def convert_timestamp(value,
|
112
|
+
def convert_timestamp(value, _definition)
|
115
113
|
# the time value can be in 2 formats:
|
116
114
|
# - 2010-10-26 02:27:15 +0000 (ruby 1.9.2)
|
117
115
|
# - Thu Oct 25 16:15:18 -0800 2010
|
118
116
|
if value =~ /^#{CommonRegularExpressions::TIMESTAMP_PARTS['Y']}/
|
119
|
-
value.gsub!(/\W/,'')
|
117
|
+
value.gsub!(/\W/, '')
|
120
118
|
value[0..13].to_i
|
121
119
|
else
|
122
|
-
value.gsub!(/\W/,'')
|
120
|
+
value.gsub!(/\W/, '')
|
123
121
|
time_as_str = value[-4..-1] # year
|
124
122
|
# convert the month to a 2-digit representation
|
125
|
-
month = MONTHS.index(value[3..5])+1
|
123
|
+
month = MONTHS.index(value[3..5]) + 1
|
126
124
|
month < 10 ? time_as_str << "0#{month}" : time_as_str << month.to_s
|
127
125
|
|
128
126
|
time_as_str << value[6..13] # day of month + time
|
129
127
|
time_as_str.to_i
|
130
128
|
end
|
131
|
-
|
132
129
|
end
|
133
130
|
end
|
134
|
-
|
135
131
|
end
|
136
132
|
end
|