request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# PostgresQL spec 8.3.7
|
|
4
3
|
class Postgresql < Base
|
|
5
|
-
|
|
6
4
|
extend CommonRegularExpressions
|
|
7
|
-
|
|
5
|
+
|
|
8
6
|
line_definition :query do |line|
|
|
9
7
|
line.header = true
|
|
10
8
|
line.teaser = /.*LOG\:/
|
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%d %k:%M:%S')})\ \S+ \[\d+\]\:\ \[.*\]\ LOG\:\ \ \d+\:\ duration\: (.*)\ ms\ \ statement:\ (.*)/
|
|
12
10
|
|
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
|
14
|
-
line.capture(:query_time).as(:duration, :
|
|
12
|
+
line.capture(:query_time).as(:duration, unit: :sec)
|
|
15
13
|
line.capture(:query_fragment)
|
|
16
14
|
end
|
|
17
|
-
|
|
15
|
+
|
|
18
16
|
line_definition :location do |line|
|
|
19
17
|
line.footer = true
|
|
20
18
|
line.teaser = /.*LOCATION:/
|
|
@@ -22,7 +20,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
22
20
|
|
|
23
21
|
line.capture(:query).as(:sql) # Hack to gather up fragments
|
|
24
22
|
end
|
|
25
|
-
|
|
23
|
+
|
|
26
24
|
line_definition :query_fragment do |line|
|
|
27
25
|
line.regexp = /^(?!.*LOG)\s*(.*)\s*/
|
|
28
26
|
line.capture(:query_fragment)
|
|
@@ -30,14 +28,12 @@ module RequestLogAnalyzer::FileFormat
|
|
|
30
28
|
|
|
31
29
|
report do |analyze|
|
|
32
30
|
analyze.timespan
|
|
33
|
-
analyze.hourly_spread
|
|
34
|
-
analyze.duration :query_time, :
|
|
31
|
+
analyze.hourly_spread
|
|
32
|
+
analyze.duration :query_time, category: :query, title: 'Query time'
|
|
35
33
|
end
|
|
36
|
-
|
|
37
|
-
class Request < RequestLogAnalyzer::Request
|
|
38
34
|
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
class Request < RequestLogAnalyzer::Request
|
|
36
|
+
def convert_sql(value, _definition)
|
|
41
37
|
# Recreate the full SQL query by joining all the previous parts and this last line
|
|
42
38
|
sql = every(:query_fragment).join("\n") + value
|
|
43
39
|
|
|
@@ -52,7 +48,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
52
48
|
sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list
|
|
53
49
|
sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list
|
|
54
50
|
|
|
55
|
-
|
|
51
|
+
sql.lstrip.rstrip
|
|
56
52
|
end
|
|
57
53
|
|
|
58
54
|
def host
|
|
@@ -60,9 +56,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
60
56
|
end
|
|
61
57
|
|
|
62
58
|
# Convert the timestamp to an integer
|
|
63
|
-
def convert_timestamp(value,
|
|
59
|
+
def convert_timestamp(value, _definition)
|
|
64
60
|
_, y, m, d, h, i, s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
|
65
|
-
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
|
61
|
+
('20%s%s%s%s%s%s' % [y, m, d, h.rjust(2, '0'), i, s]).to_i
|
|
66
62
|
end
|
|
67
63
|
end
|
|
68
64
|
end
|
|
@@ -1,79 +1,77 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# Default FileFormat class for Rails logs.
|
|
4
3
|
#
|
|
5
4
|
# Instances will be created dynamically based on the lines you want it to parse. You can
|
|
6
5
|
# specify what lines should be included in the parser by providing a list to the create
|
|
7
6
|
# method as first argument.
|
|
8
7
|
class Rails < Base
|
|
9
|
-
|
|
10
8
|
extend CommonRegularExpressions
|
|
11
9
|
|
|
12
10
|
# Creates a Rails FileFormat instance.
|
|
13
11
|
#
|
|
14
|
-
# The lines that will be parsed can be defined by the argument to this function,
|
|
12
|
+
# The lines that will be parsed can be defined by the argument to this function,
|
|
15
13
|
# which should be an array of line names, or a list of line names as comma separated
|
|
16
|
-
# string. The resulting report depends on the lines that will be parsed. You can
|
|
14
|
+
# string. The resulting report depends on the lines that will be parsed. You can
|
|
17
15
|
# also provide s string that describes a common set of lines, like "production",
|
|
18
16
|
# "development" or "production".
|
|
19
17
|
def self.create(lines = 'production')
|
|
20
18
|
definitions_hash = line_definer.line_definitions.clone
|
|
21
|
-
|
|
22
|
-
lines = lines.to_s.split(',') if lines.
|
|
23
|
-
lines = [lines.to_s] if lines.
|
|
24
|
-
|
|
19
|
+
|
|
20
|
+
lines = lines.to_s.split(',') if lines.is_a?(String)
|
|
21
|
+
lines = [lines.to_s] if lines.is_a?(Symbol)
|
|
22
|
+
|
|
25
23
|
lines.each do |line|
|
|
26
24
|
line = line.to_sym
|
|
27
|
-
if LINE_COLLECTIONS.
|
|
25
|
+
if LINE_COLLECTIONS.key?(line)
|
|
28
26
|
LINE_COLLECTIONS[line].each { |l| definitions_hash[l] ||= LINE_DEFINITIONS[l] }
|
|
29
|
-
elsif LINE_DEFINITIONS.
|
|
27
|
+
elsif LINE_DEFINITIONS.key?(line)
|
|
30
28
|
definitions_hash[line] ||= LINE_DEFINITIONS[line]
|
|
31
29
|
else
|
|
32
|
-
|
|
30
|
+
fail "Unrecognized Rails log line name: #{line.inspect}!"
|
|
33
31
|
end
|
|
34
32
|
end
|
|
35
33
|
|
|
36
|
-
|
|
34
|
+
new(definitions_hash, report_trackers(definitions_hash))
|
|
37
35
|
end
|
|
38
|
-
|
|
36
|
+
|
|
39
37
|
# Creates trackers based on the specified line definitions.
|
|
40
38
|
#
|
|
41
39
|
# The more lines that will be parsed, the more information will appear in the report.
|
|
42
40
|
def self.report_trackers(lines)
|
|
43
41
|
analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
|
|
44
|
-
|
|
42
|
+
|
|
45
43
|
analyze.timespan
|
|
46
44
|
analyze.hourly_spread
|
|
47
|
-
|
|
48
|
-
analyze.frequency :
|
|
49
|
-
analyze.frequency :method, :
|
|
50
|
-
analyze.frequency :status, :
|
|
51
|
-
|
|
52
|
-
if lines.
|
|
53
|
-
analyze.frequency(:
|
|
54
|
-
|
|
45
|
+
|
|
46
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Most requested'
|
|
47
|
+
analyze.frequency :method, title: 'HTTP methods'
|
|
48
|
+
analyze.frequency :status, title: 'HTTP statuses returned'
|
|
49
|
+
|
|
50
|
+
if lines.key?(:cache_hit)
|
|
51
|
+
analyze.frequency(category: lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' },
|
|
52
|
+
title: 'Rails action cache hits')
|
|
55
53
|
end
|
|
56
|
-
|
|
57
|
-
analyze.duration :duration, :
|
|
58
|
-
analyze.duration :view, :
|
|
59
|
-
analyze.duration :db, :
|
|
60
|
-
|
|
61
|
-
analyze.frequency :
|
|
62
|
-
:
|
|
63
|
-
|
|
64
|
-
if lines.
|
|
65
|
-
analyze.frequency :error, :
|
|
54
|
+
|
|
55
|
+
analyze.duration :duration, category: REQUEST_CATEGORIZER, title: 'Request duration', line_type: :completed
|
|
56
|
+
analyze.duration :view, category: REQUEST_CATEGORIZER, title: 'View rendering time', line_type: :completed
|
|
57
|
+
analyze.duration :db, category: REQUEST_CATEGORIZER, title: 'Database time', line_type: :completed
|
|
58
|
+
|
|
59
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Process blockers (> 1 sec duration)',
|
|
60
|
+
if: lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
|
61
|
+
|
|
62
|
+
if lines.key?(:failure)
|
|
63
|
+
analyze.frequency :error, title: 'Failed requests', line_type: :failure
|
|
66
64
|
end
|
|
67
65
|
|
|
68
|
-
if lines.
|
|
69
|
-
analyze.duration :render_duration, :
|
|
66
|
+
if lines.key?(:rendered)
|
|
67
|
+
analyze.duration :render_duration, category: :render_file, multiple: true, title: 'Partial rendering duration'
|
|
70
68
|
end
|
|
71
69
|
|
|
72
|
-
if lines.
|
|
73
|
-
analyze.duration :query_duration, :
|
|
70
|
+
if lines.key?(:query_executed)
|
|
71
|
+
analyze.duration :query_duration, category: :query_sql, multiple: true, title: 'Query duration'
|
|
74
72
|
end
|
|
75
|
-
|
|
76
|
-
|
|
73
|
+
|
|
74
|
+
analyze.trackers + report_definer.trackers
|
|
77
75
|
end
|
|
78
76
|
|
|
79
77
|
# Rails < 2.1 completed line example
|
|
@@ -83,92 +81,90 @@ module RequestLogAnalyzer::FileFormat
|
|
|
83
81
|
# Rails > 2.1 completed line example
|
|
84
82
|
# Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
|
|
85
83
|
RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+))?,?(?:.?DB: (\d+))?\)? \| (\d{3}).+\[(http.+)\]/
|
|
86
|
-
|
|
84
|
+
|
|
87
85
|
# A hash of definitions for all common lines in Rails logs.
|
|
88
86
|
LINE_DEFINITIONS = {
|
|
89
|
-
:
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
:
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
:
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
:
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
:
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
:
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
:
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
:
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
87
|
+
processing: RequestLogAnalyzer::LineDefinition.new(:processing, header: true,
|
|
88
|
+
teaser: /Processing /,
|
|
89
|
+
regexp: /Processing ((?:\w+::)*\w+)#(\w+)(?: to (\w+))? \(for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})\) \[([A-Z]+)\]/,
|
|
90
|
+
captures: [{ name: :controller, type: :string },
|
|
91
|
+
{ name: :action, type: :string },
|
|
92
|
+
{ name: :format, type: :string, default: 'html' },
|
|
93
|
+
{ name: :ip, type: :string },
|
|
94
|
+
{ name: :timestamp, type: :timestamp },
|
|
95
|
+
{ name: :method, type: :string }]),
|
|
96
|
+
|
|
97
|
+
completed: RequestLogAnalyzer::LineDefinition.new(:completed, footer: true,
|
|
98
|
+
teaser: /Completed in /,
|
|
99
|
+
regexp: Regexp.union(RAILS_21_COMPLETED, RAILS_22_COMPLETED),
|
|
100
|
+
captures: [{ name: :duration, type: :duration, unit: :sec }, # First old variant capture
|
|
101
|
+
{ name: :view, type: :duration, unit: :sec },
|
|
102
|
+
{ name: :db, type: :duration, unit: :sec },
|
|
103
|
+
{ name: :status, type: :integer },
|
|
104
|
+
{ name: :url, type: :string }, # Last old variant capture
|
|
105
|
+
{ name: :duration, type: :duration, unit: :msec }, # First new variant capture
|
|
106
|
+
{ name: :view, type: :duration, unit: :msec },
|
|
107
|
+
{ name: :db, type: :duration, unit: :msec },
|
|
108
|
+
{ name: :status, type: :integer },
|
|
109
|
+
{ name: :url, type: :string }]), # Last new variant capture
|
|
110
|
+
|
|
111
|
+
failure: RequestLogAnalyzer::LineDefinition.new(:failure, footer: true,
|
|
112
|
+
teaser: /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:/,
|
|
113
|
+
regexp: /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/,
|
|
114
|
+
captures: [{ name: :error, type: :string },
|
|
115
|
+
{ name: :message, type: :string },
|
|
116
|
+
{ name: :line, type: :integer },
|
|
117
|
+
{ name: :file, type: :string }]),
|
|
118
|
+
|
|
119
|
+
cache_hit: RequestLogAnalyzer::LineDefinition.new(:cache_hit,
|
|
120
|
+
regexp: /Filter chain halted as \[\#<ActionController::Filters::AroundFilter.*\@method=.*(?:Caching::Actions::ActionCacheFilter|action_controller\/caching\/actions\.rb).*\] did_not_yield/),
|
|
121
|
+
|
|
122
|
+
parameters: RequestLogAnalyzer::LineDefinition.new(:parameters,
|
|
123
|
+
teaser: / Parameters:/,
|
|
124
|
+
regexp: / Parameters:\s+(\{.*\})/,
|
|
125
|
+
captures: [{ name: :params, type: :eval }]),
|
|
126
|
+
|
|
127
|
+
rendered: RequestLogAnalyzer::LineDefinition.new(:rendered,
|
|
128
|
+
teaser: /Rendered /,
|
|
129
|
+
regexp: /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
|
|
130
|
+
captures: [{ name: :render_file, type: :string },
|
|
131
|
+
{ name: :render_duration, type: :duration, unit: :msec }]),
|
|
132
|
+
|
|
133
|
+
query_executed: RequestLogAnalyzer::LineDefinition.new(:query_executed,
|
|
134
|
+
regexp: /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/,
|
|
135
|
+
captures: [{ name: :query_class, type: :string },
|
|
136
|
+
{ name: :query_duration, type: :duration, unit: :msec },
|
|
137
|
+
{ name: :query_sql, type: :sql }]),
|
|
138
|
+
|
|
139
|
+
query_cached: RequestLogAnalyzer::LineDefinition.new(:query_cached,
|
|
140
|
+
regexp: /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/,
|
|
141
|
+
captures: [{ name: :cached_duration, type: :duration, unit: :msec },
|
|
142
|
+
{ name: :cached_sql, type: :sql }])
|
|
145
143
|
}
|
|
146
144
|
|
|
147
145
|
# Definitions of common combinations of lines that can be parsed
|
|
148
146
|
LINE_COLLECTIONS = {
|
|
149
|
-
:
|
|
150
|
-
:
|
|
151
|
-
:
|
|
152
|
-
:
|
|
147
|
+
minimal: [:processing, :completed],
|
|
148
|
+
production: [:processing, :completed, :failure, :cache_hit],
|
|
149
|
+
development: [:processing, :completed, :failure, :rendered, :query_executed, :query_cached],
|
|
150
|
+
all: LINE_DEFINITIONS.keys
|
|
153
151
|
}
|
|
154
152
|
|
|
155
|
-
|
|
156
153
|
# Simple function to categorize Rails requests using controller/actions/format and method.
|
|
157
|
-
REQUEST_CATEGORIZER =
|
|
154
|
+
REQUEST_CATEGORIZER = proc do |request|
|
|
158
155
|
"#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
|
|
159
156
|
end
|
|
160
157
|
|
|
161
158
|
# Define a custom Request class for the Rails file format to speed up timestamp handling
|
|
162
159
|
# and to ensure that a format is always set.
|
|
163
160
|
class Request < RequestLogAnalyzer::Request
|
|
164
|
-
|
|
165
161
|
# Do not use DateTime.parse
|
|
166
|
-
def convert_timestamp(value,
|
|
162
|
+
def convert_timestamp(value, _definition)
|
|
167
163
|
value.gsub(/[^0-9]/, '')[0...14].to_i
|
|
168
164
|
end
|
|
169
165
|
|
|
170
166
|
# Sanitizes SQL queries so that they can be grouped
|
|
171
|
-
def convert_sql(sql,
|
|
167
|
+
def convert_sql(sql, _definition)
|
|
172
168
|
sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
|
|
173
169
|
end
|
|
174
170
|
end
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# Default FileFormat class for Rails 3 logs.
|
|
4
3
|
#
|
|
5
4
|
# For now, this is just a basic implementation. It will probaby change after
|
|
6
5
|
# Rails 3 final has been released.
|
|
7
6
|
class Rails3 < Base
|
|
8
|
-
|
|
9
7
|
extend CommonRegularExpressions
|
|
10
8
|
|
|
11
9
|
# beta4: Started GET "/" for 127.0.0.1 at Wed Jul 07 09:13:27 -0700 2010 (different time format)
|
|
@@ -46,9 +44,9 @@ module RequestLogAnalyzer::FileFormat
|
|
|
46
44
|
line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
|
|
47
45
|
|
|
48
46
|
line.capture(:status).as(:integer)
|
|
49
|
-
line.capture(:duration).as(:duration, :
|
|
50
|
-
line.capture(:view).as(:duration, :
|
|
51
|
-
line.capture(:db).as(:duration, :
|
|
47
|
+
line.capture(:duration).as(:duration, unit: :msec)
|
|
48
|
+
line.capture(:view).as(:duration, unit: :msec)
|
|
49
|
+
line.capture(:db).as(:duration, unit: :msec)
|
|
52
50
|
end
|
|
53
51
|
|
|
54
52
|
# ActionController::RoutingError (No route matches [GET] "/missing_stuff"):
|
|
@@ -76,7 +74,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
76
74
|
line.teaser = /\bRendered /
|
|
77
75
|
line.regexp = /\bRendered ([a-zA-Z0-9_\-\/.]+(?:\/[a-zA-Z0-9_\-.]+)+)(?:\ within\ .*?)? \((\d+(?:\.\d+)?)ms\)/
|
|
78
76
|
line.capture(:rendered_file)
|
|
79
|
-
line.capture(:partial_duration).as(:duration, :
|
|
77
|
+
line.capture(:partial_duration).as(:duration, unit: :msec)
|
|
80
78
|
end
|
|
81
79
|
|
|
82
80
|
# # Not parsed at the moment:
|
|
@@ -91,46 +89,44 @@ module RequestLogAnalyzer::FileFormat
|
|
|
91
89
|
analyze.timespan
|
|
92
90
|
analyze.hourly_spread
|
|
93
91
|
|
|
94
|
-
analyze.frequency :
|
|
95
|
-
analyze.frequency :method, :
|
|
96
|
-
analyze.frequency :status, :
|
|
92
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Most requested'
|
|
93
|
+
analyze.frequency :method, title: 'HTTP methods'
|
|
94
|
+
analyze.frequency :status, title: 'HTTP statuses returned'
|
|
97
95
|
|
|
98
|
-
analyze.duration :duration, :
|
|
99
|
-
analyze.duration :partial_duration, :
|
|
100
|
-
analyze.duration :view, :
|
|
101
|
-
analyze.duration :db, :
|
|
96
|
+
analyze.duration :duration, category: REQUEST_CATEGORIZER, title: 'Request duration', line_type: :completed
|
|
97
|
+
analyze.duration :partial_duration, category: :rendered_file, title: 'Partials rendering time', line_type: :rendered
|
|
98
|
+
analyze.duration :view, category: REQUEST_CATEGORIZER, title: 'View rendering time', line_type: :completed
|
|
99
|
+
analyze.duration :db, category: REQUEST_CATEGORIZER, title: 'Database time', line_type: :completed
|
|
102
100
|
|
|
103
|
-
analyze.frequency :
|
|
104
|
-
:
|
|
101
|
+
analyze.frequency category: REQUEST_CATEGORIZER, title: 'Process blockers (> 1 sec duration)',
|
|
102
|
+
if: lambda { |request| request[:duration] && request[:duration] > 1.0 }
|
|
105
103
|
|
|
106
|
-
analyze.frequency :
|
|
107
|
-
:
|
|
104
|
+
analyze.frequency category: lambda { |x| "[#{x[:missing_resource_method]}] #{x[:missing_resource]}" },
|
|
105
|
+
title: 'Routing Errors', if: lambda { |request| !request[:missing_resource].nil? }
|
|
108
106
|
end
|
|
109
107
|
|
|
110
108
|
class Request < RequestLogAnalyzer::Request
|
|
111
109
|
# Used to handle conversion of abbrev. month name to a digit
|
|
112
110
|
MONTHS = %w(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
|
|
113
111
|
|
|
114
|
-
def convert_timestamp(value,
|
|
112
|
+
def convert_timestamp(value, _definition)
|
|
115
113
|
# the time value can be in 2 formats:
|
|
116
114
|
# - 2010-10-26 02:27:15 +0000 (ruby 1.9.2)
|
|
117
115
|
# - Thu Oct 25 16:15:18 -0800 2010
|
|
118
116
|
if value =~ /^#{CommonRegularExpressions::TIMESTAMP_PARTS['Y']}/
|
|
119
|
-
value.gsub!(/\W/,'')
|
|
117
|
+
value.gsub!(/\W/, '')
|
|
120
118
|
value[0..13].to_i
|
|
121
119
|
else
|
|
122
|
-
value.gsub!(/\W/,'')
|
|
120
|
+
value.gsub!(/\W/, '')
|
|
123
121
|
time_as_str = value[-4..-1] # year
|
|
124
122
|
# convert the month to a 2-digit representation
|
|
125
|
-
month = MONTHS.index(value[3..5])+1
|
|
123
|
+
month = MONTHS.index(value[3..5]) + 1
|
|
126
124
|
month < 10 ? time_as_str << "0#{month}" : time_as_str << month.to_s
|
|
127
125
|
|
|
128
126
|
time_as_str << value[6..13] # day of month + time
|
|
129
127
|
time_as_str.to_i
|
|
130
128
|
end
|
|
131
|
-
|
|
132
129
|
end
|
|
133
130
|
end
|
|
134
|
-
|
|
135
131
|
end
|
|
136
132
|
end
|