request-log-analyzer 1.13.1 → 1.13.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
@@ -1,11 +1,9 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# FileFormat for Amazon S3 access logs.
|
4
3
|
#
|
5
4
|
# Access logs are disabled by default on Amazon S3. To enable logging, see
|
6
5
|
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
|
7
6
|
class AmazonS3 < Base
|
8
|
-
|
9
7
|
extend CommonRegularExpressions
|
10
8
|
|
11
9
|
line_definition :access do |line|
|
@@ -24,10 +22,10 @@ module RequestLogAnalyzer::FileFormat
|
|
24
22
|
line.capture(:request_uri)
|
25
23
|
line.capture(:http_status).as(:integer)
|
26
24
|
line.capture(:error_code).as(:nillable_string)
|
27
|
-
line.capture(:bytes_sent).as(:traffic, :
|
28
|
-
line.capture(:object_size).as(:traffic, :
|
29
|
-
line.capture(:total_time).as(:duration, :
|
30
|
-
line.capture(:turnaround_time).as(:duration, :
|
25
|
+
line.capture(:bytes_sent).as(:traffic, unit: :byte)
|
26
|
+
line.capture(:object_size).as(:traffic, unit: :byte)
|
27
|
+
line.capture(:total_time).as(:duration, unit: :msec)
|
28
|
+
line.capture(:turnaround_time).as(:duration, unit: :msec)
|
31
29
|
line.capture(:referer).as(:referer)
|
32
30
|
line.capture(:user_agent).as(:user_agent)
|
33
31
|
end
|
@@ -36,39 +34,37 @@ module RequestLogAnalyzer::FileFormat
|
|
36
34
|
analyze.timespan
|
37
35
|
analyze.hourly_spread
|
38
36
|
|
39
|
-
analyze.frequency :
|
40
|
-
analyze.duration :
|
41
|
-
analyze.traffic
|
42
|
-
analyze.frequency :
|
43
|
-
analyze.frequency :
|
37
|
+
analyze.frequency category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Most popular items'
|
38
|
+
analyze.duration duration: :total_time, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Request duration'
|
39
|
+
analyze.traffic traffic: :bytes_sent, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Traffic'
|
40
|
+
analyze.frequency category: :http_status, title: 'HTTP status codes'
|
41
|
+
analyze.frequency category: :error_code, title: 'Error codes'
|
44
42
|
end
|
45
43
|
|
46
44
|
class Request < RequestLogAnalyzer::Request
|
47
|
-
|
48
|
-
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
45
|
+
MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
49
46
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
50
47
|
|
51
48
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
52
49
|
# to speed up parsing.
|
53
|
-
def convert_timestamp(value,
|
54
|
-
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
50
|
+
def convert_timestamp(value, _definition)
|
51
|
+
"#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
|
55
52
|
end
|
56
53
|
|
57
54
|
# Make sure that the string '-' is parsed as a nil value.
|
58
|
-
def convert_nillable_string(value,
|
55
|
+
def convert_nillable_string(value, _definition)
|
59
56
|
value == '-' ? nil : value
|
60
57
|
end
|
61
58
|
|
62
59
|
# Can be implemented in subclasses for improved categorizations
|
63
|
-
def convert_referer(value,
|
60
|
+
def convert_referer(value, _definition)
|
64
61
|
value == '-' ? nil : value
|
65
62
|
end
|
66
63
|
|
67
64
|
# Can be implemented in subclasses for improved categorizations
|
68
|
-
def convert_user_agent(value,
|
65
|
+
def convert_user_agent(value, _definition)
|
69
66
|
value == '-' ? nil : value
|
70
67
|
end
|
71
68
|
end
|
72
|
-
|
73
69
|
end
|
74
70
|
end
|
@@ -1,5 +1,4 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The Apache file format is able to log Apache access.log files.
|
4
3
|
#
|
5
4
|
# The access.log can be configured in Apache to have many different formats. In theory, this
|
@@ -13,18 +12,17 @@ module RequestLogAnalyzer::FileFormat
|
|
13
12
|
# From the command line, you can provide the format string using the <tt>--apache-format</tt>
|
14
13
|
# command line option.
|
15
14
|
class Apache < Base
|
16
|
-
|
17
15
|
extend CommonRegularExpressions
|
18
16
|
|
19
17
|
# A hash of predefined Apache log formats
|
20
18
|
LOG_FORMAT_DEFAULTS = {
|
21
|
-
:
|
22
|
-
:
|
23
|
-
:
|
24
|
-
:
|
25
|
-
:
|
26
|
-
:
|
27
|
-
:
|
19
|
+
common: '%h %l %u %t "%r" %>s %b',
|
20
|
+
combined: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
|
21
|
+
vhost_combined: '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
|
22
|
+
nginx: '%a %t %h %u "%r" %>s %b',
|
23
|
+
rack: '%h %l %u %t "%r" %>s %b %T',
|
24
|
+
referer: '%{Referer}i -> %U',
|
25
|
+
agent: '%{User-agent}i'
|
28
26
|
}
|
29
27
|
|
30
28
|
# I have encountered two timestamp types, with timezone and without. Parse both.
|
@@ -32,26 +30,26 @@ module RequestLogAnalyzer::FileFormat
|
|
32
30
|
|
33
31
|
# A hash that defines how the log format directives should be parsed.
|
34
32
|
LOG_DIRECTIVES = {
|
35
|
-
'%' => { nil => { :
|
36
|
-
'v' => { nil => { :
|
37
|
-
'h' => { nil => { :
|
38
|
-
'a' => { nil => { :
|
39
|
-
'b' => { nil => { :
|
40
|
-
'c' => { nil => { :
|
41
|
-
'D' => { nil => { :
|
42
|
-
'micro' => { :
|
43
|
-
'milli' => { :
|
33
|
+
'%' => { nil => { regexp: '%', captures: [] } },
|
34
|
+
'v' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :vhost, type: :string }] } },
|
35
|
+
'h' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :remote_host, type: :string }] } },
|
36
|
+
'a' => { nil => { regexp: "(#{ip_address})", captures: [{ name: :remote_ip, type: :string }] } },
|
37
|
+
'b' => { nil => { regexp: '(\d+|-)', captures: [{ name: :bytes_sent, type: :traffic }] } },
|
38
|
+
'c' => { nil => { regexp: '(\+|\-|\X)', captures: [{ name: :connection_status, type: :integer }] } },
|
39
|
+
'D' => { nil => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
|
40
|
+
'micro' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
|
41
|
+
'milli' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :msec }] }
|
44
42
|
},
|
45
|
-
'l' => { nil => { :
|
46
|
-
'T' => { nil => { :
|
47
|
-
't' => { nil => { :
|
48
|
-
's' => { nil => { :
|
49
|
-
'u' => { nil => { :
|
50
|
-
'U' => { nil => { :
|
51
|
-
'r' => { nil => { :
|
52
|
-
|
53
|
-
'i' => { 'Referer' => { :
|
54
|
-
'User-agent' => { :
|
43
|
+
'l' => { nil => { regexp: '([\w-]+)', captures: [{ name: :remote_logname, type: :nillable_string }] } },
|
44
|
+
'T' => { nil => { regexp: '(\d+(?:\.\d+)?|-)', captures: [{ name: :duration, type: :duration, unit: :sec }] } },
|
45
|
+
't' => { nil => { regexp: "\\[(#{APACHE_TIMESTAMP})?\\]", captures: [{ name: :timestamp, type: :timestamp }] } },
|
46
|
+
's' => { nil => { regexp: '(\d{3})', captures: [{ name: :http_status, type: :integer }] } },
|
47
|
+
'u' => { nil => { regexp: '(\/\S*|-)', captures: [{ name: :user, type: :nillable_string }] } },
|
48
|
+
'U' => { nil => { regexp: '(\/\S*)', captures: [{ name: :path, type: :string }] } },
|
49
|
+
'r' => { nil => { regexp: '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', captures: [{ name: :http_method, type: :string },
|
50
|
+
{ name: :path, type: :path }, { name: :http_version, type: :string }] } },
|
51
|
+
'i' => { 'Referer' => { regexp: '(\S+)', captures: [{ name: :referer, type: :nillable_string }] },
|
52
|
+
'User-agent' => { regexp: '(.*)', captures: [{ name: :user_agent, type: :user_agent }] }
|
55
53
|
}
|
56
54
|
}
|
57
55
|
|
@@ -61,7 +59,7 @@ module RequestLogAnalyzer::FileFormat
|
|
61
59
|
def self.create(*args)
|
62
60
|
access_line = access_line_definition(args.first)
|
63
61
|
trackers = report_trackers(access_line) + report_definer.trackers
|
64
|
-
|
62
|
+
new(line_definer.line_definitions.merge(access: access_line), trackers)
|
65
63
|
end
|
66
64
|
|
67
65
|
# Creates the access log line definition based on the Apache log format string
|
@@ -90,8 +88,8 @@ module RequestLogAnalyzer::FileFormat
|
|
90
88
|
end
|
91
89
|
|
92
90
|
# Return a new line definition object
|
93
|
-
|
94
|
-
:
|
91
|
+
RequestLogAnalyzer::LineDefinition.new(:access, regexp: Regexp.new(line_regexp),
|
92
|
+
captures: captures, header: true, footer: true)
|
95
93
|
end
|
96
94
|
|
97
95
|
# Sets up the report trackers according to the fields captured by the access line definition.
|
@@ -101,40 +99,39 @@ module RequestLogAnalyzer::FileFormat
|
|
101
99
|
analyze.timespan if line_definition.captures?(:timestamp)
|
102
100
|
analyze.hourly_spread if line_definition.captures?(:timestamp)
|
103
101
|
|
104
|
-
analyze.frequency :
|
105
|
-
analyze.frequency :
|
106
|
-
analyze.frequency :
|
102
|
+
analyze.frequency category: :http_method, title: 'HTTP methods' if line_definition.captures?(:http_method)
|
103
|
+
analyze.frequency category: :http_status, title: 'HTTP statuses' if line_definition.captures?(:http_status)
|
104
|
+
analyze.frequency category: lambda { |r| r.category }, title: 'Most popular URIs' if line_definition.captures?(:path)
|
107
105
|
|
108
|
-
analyze.frequency :
|
109
|
-
analyze.frequency :
|
106
|
+
analyze.frequency category: :user_agent, title: 'User agents' if line_definition.captures?(:user_agent)
|
107
|
+
analyze.frequency category: :referer, title: 'Referers' if line_definition.captures?(:referer)
|
110
108
|
|
111
|
-
analyze.duration :
|
112
|
-
analyze.traffic
|
109
|
+
analyze.duration duration: :duration, category: lambda { |r| r.category }, title: 'Request duration' if line_definition.captures?(:duration)
|
110
|
+
analyze.traffic traffic: :bytes_sent, category: lambda { |r| r.category }, title: 'Traffic' if line_definition.captures?(:bytes_sent)
|
113
111
|
|
114
|
-
|
112
|
+
analyze.trackers
|
115
113
|
end
|
116
114
|
|
117
115
|
# Define a custom Request class for the Apache file format to speed up timestamp handling.
|
118
116
|
class Request < RequestLogAnalyzer::Request
|
119
|
-
|
120
117
|
def category
|
121
118
|
first(:path)
|
122
119
|
end
|
123
120
|
|
124
|
-
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
121
|
+
MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
125
122
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
126
123
|
|
127
124
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
128
125
|
# to speed up parsing.
|
129
|
-
def convert_timestamp(value,
|
130
|
-
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
126
|
+
def convert_timestamp(value, _definition)
|
127
|
+
"#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
|
131
128
|
end
|
132
129
|
|
133
130
|
# This function can be overridden to simplify the user agent string for better
|
134
131
|
# categorization in the reports
|
135
|
-
def convert_user_agent(value,
|
132
|
+
def convert_user_agent(value, _definition)
|
136
133
|
value # TODO
|
137
134
|
end
|
138
135
|
end
|
139
136
|
end
|
140
|
-
end
|
137
|
+
end
|
@@ -1,40 +1,38 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The DelayedJob file format parsed log files that are created by DelayedJob.
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
5
4
|
class DelayedJob < Base
|
6
|
-
|
7
5
|
line_definition :job_lock do |line|
|
8
6
|
line.header = true
|
9
7
|
line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
|
10
|
-
|
8
|
+
|
11
9
|
line.capture(:job)
|
12
10
|
end
|
13
|
-
|
11
|
+
|
14
12
|
line_definition :job_completed do |line|
|
15
13
|
line.footer = true
|
16
14
|
line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
|
17
15
|
|
18
16
|
line.capture(:completed_job)
|
19
|
-
line.capture(:duration).as(:duration, :
|
17
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
20
18
|
end
|
21
|
-
|
19
|
+
|
22
20
|
line_definition :job_failed do |line|
|
23
21
|
line.footer = true
|
24
22
|
line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
|
25
|
-
|
23
|
+
|
26
24
|
line.capture(:failed_job)
|
27
25
|
line.capture(:exception)
|
28
26
|
line.capture(:attempts).as(:integer)
|
29
27
|
end
|
30
|
-
|
28
|
+
|
31
29
|
line_definition :job_lock_failed do |line|
|
32
30
|
line.footer = true
|
33
31
|
line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
|
34
|
-
|
32
|
+
|
35
33
|
line.capture(:locked_job)
|
36
34
|
end
|
37
|
-
|
35
|
+
|
38
36
|
# line_definition :batch_completed do |line|
|
39
37
|
# line.header = true
|
40
38
|
# line.footer = true
|
@@ -44,12 +42,12 @@ module RequestLogAnalyzer::FileFormat
|
|
44
42
|
# line.capture(:mean_duration).as(:duration, :unit => :sec)
|
45
43
|
# line.capture(:failed_amount).as(:integer)
|
46
44
|
# end
|
47
|
-
|
45
|
+
|
48
46
|
report do |analyze|
|
49
|
-
analyze.frequency :job, :
|
50
|
-
analyze.frequency :job, :
|
51
|
-
|
52
|
-
analyze.duration :duration, :
|
47
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
48
|
+
analyze.frequency :job, if: lambda { |request| request[:attempts] == 1 }, title: 'Failed jobs'
|
49
|
+
|
50
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
53
51
|
end
|
54
52
|
end
|
55
53
|
end
|
@@ -1,38 +1,36 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The DelayedJob2 file format parsed log files that are created by DelayedJob 2.0.
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
5
4
|
class DelayedJob2 < Base
|
6
|
-
|
7
5
|
extend CommonRegularExpressions
|
8
|
-
|
6
|
+
|
9
7
|
line_definition :job_lock do |line|
|
10
8
|
line.header = true
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
12
|
-
|
10
|
+
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
14
12
|
line.capture(:host)
|
15
13
|
line.capture(:pid).as(:integer)
|
16
14
|
line.capture(:job)
|
17
15
|
end
|
18
|
-
|
16
|
+
|
19
17
|
line_definition :job_completed do |line|
|
20
18
|
line.footer = true
|
21
19
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
|
22
20
|
line.capture(:timestamp).as(:timestamp)
|
23
21
|
line.capture(:host)
|
24
22
|
line.capture(:pid).as(:integer)
|
25
|
-
line.capture(:duration).as(:duration, :
|
23
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
26
24
|
end
|
27
|
-
|
25
|
+
|
28
26
|
report do |analyze|
|
29
27
|
analyze.timespan
|
30
28
|
analyze.hourly_spread
|
31
29
|
|
32
|
-
analyze.frequency :job, :
|
33
|
-
#analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
34
|
-
|
35
|
-
analyze.duration :duration, :
|
30
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
31
|
+
# analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
32
|
+
|
33
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
36
34
|
end
|
37
35
|
end
|
38
36
|
end
|
@@ -1,21 +1,19 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The DelayedJob21 file format parsed log files that are created by DelayedJob 2.1 or higher.
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
5
4
|
class DelayedJob21 < Base
|
6
|
-
|
7
5
|
extend CommonRegularExpressions
|
8
|
-
|
6
|
+
|
9
7
|
line_definition :job_lock do |line|
|
10
8
|
line.header = true
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
12
|
-
|
10
|
+
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
14
12
|
line.capture(:host)
|
15
13
|
line.capture(:pid).as(:integer)
|
16
14
|
line.capture(:job)
|
17
15
|
end
|
18
|
-
|
16
|
+
|
19
17
|
line_definition :job_completed do |line|
|
20
18
|
line.footer = true
|
21
19
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
|
@@ -23,17 +21,17 @@ module RequestLogAnalyzer::FileFormat
|
|
23
21
|
line.capture(:host)
|
24
22
|
line.capture(:pid).as(:integer)
|
25
23
|
line.capture(:job)
|
26
|
-
line.capture(:duration).as(:duration, :
|
24
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
27
25
|
end
|
28
|
-
|
26
|
+
|
29
27
|
report do |analyze|
|
30
28
|
analyze.timespan
|
31
29
|
analyze.hourly_spread
|
32
30
|
|
33
|
-
analyze.frequency :job, :
|
34
|
-
#analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
35
|
-
|
36
|
-
analyze.duration :duration, :
|
31
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
32
|
+
# analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
33
|
+
|
34
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
37
35
|
end
|
38
36
|
end
|
39
37
|
end
|
@@ -1,9 +1,7 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The DelayedJob3 file format parsed log files that are created by DelayedJob 3.0 or higher.
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
5
4
|
class DelayedJob3 < Base
|
6
|
-
|
7
5
|
extend CommonRegularExpressions
|
8
6
|
|
9
7
|
line_definition :job_completed do |line|
|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
|
|
14
12
|
line.capture(:host)
|
15
13
|
line.capture(:pid).as(:integer)
|
16
14
|
line.capture(:job)
|
17
|
-
line.capture(:duration).as(:duration, :
|
15
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
18
16
|
end
|
19
17
|
|
20
18
|
line_definition :job_failed do |line|
|
@@ -39,15 +37,14 @@ module RequestLogAnalyzer::FileFormat
|
|
39
37
|
line.capture(:failures).as(:integer)
|
40
38
|
end
|
41
39
|
|
42
|
-
|
43
40
|
report do |analyze|
|
44
41
|
analyze.timespan
|
45
42
|
analyze.hourly_spread
|
46
43
|
|
47
|
-
analyze.frequency :job, :
|
48
|
-
analyze.frequency :job, :
|
49
|
-
analyze.frequency :failures, :
|
50
|
-
analyze.duration :duration, :
|
44
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
45
|
+
analyze.frequency :job, category: :job, line_type: :job_failed, title: 'Failed jobs'
|
46
|
+
analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
|
47
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
51
48
|
end
|
52
49
|
end
|
53
50
|
end
|
@@ -1,9 +1,7 @@
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
2
|
-
|
3
2
|
# The DelayedJob4 file format parsed log files that are created by DelayedJob 4.0 or higher.
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
5
4
|
class DelayedJob4 < Base
|
6
|
-
|
7
5
|
extend CommonRegularExpressions
|
8
6
|
|
9
7
|
line_definition :job_completed do |line|
|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
|
|
14
12
|
line.capture(:host)
|
15
13
|
line.capture(:pid).as(:integer)
|
16
14
|
line.capture(:job)
|
17
|
-
line.capture(:duration).as(:duration, :
|
15
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
18
16
|
end
|
19
17
|
|
20
18
|
line_definition :job_failed do |line|
|
@@ -40,15 +38,14 @@ module RequestLogAnalyzer::FileFormat
|
|
40
38
|
line.capture(:failures).as(:integer)
|
41
39
|
end
|
42
40
|
|
43
|
-
|
44
41
|
report do |analyze|
|
45
42
|
analyze.timespan
|
46
43
|
analyze.hourly_spread
|
47
44
|
|
48
|
-
analyze.frequency :job, :
|
49
|
-
analyze.frequency :job, :
|
50
|
-
analyze.frequency :failures, :
|
51
|
-
analyze.duration :duration, :
|
45
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
46
|
+
analyze.frequency :job, category: lambda { |r| "#{r[:job]} #{r[:error]}" }, line_type: :job_failed, title: 'Failed jobs'
|
47
|
+
analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
|
48
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
52
49
|
end
|
53
50
|
end
|
54
51
|
end
|