request-log-analyzer 1.13.1 → 1.13.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/bin/console +17 -0
- data/lib/cli/command_line_arguments.rb +29 -36
- data/lib/cli/database_console.rb +1 -3
- data/lib/cli/database_console_init.rb +11 -11
- data/lib/cli/progressbar.rb +30 -32
- data/lib/cli/tools.rb +20 -23
- data/lib/request_log_analyzer.rb +8 -8
- data/lib/request_log_analyzer/aggregator.rb +4 -7
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
- data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
- data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
- data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
- data/lib/request_log_analyzer/controller.rb +36 -42
- data/lib/request_log_analyzer/database.rb +4 -6
- data/lib/request_log_analyzer/database/base.rb +39 -41
- data/lib/request_log_analyzer/database/connection.rb +8 -10
- data/lib/request_log_analyzer/database/request.rb +1 -3
- data/lib/request_log_analyzer/database/source.rb +0 -2
- data/lib/request_log_analyzer/database/warning.rb +4 -6
- data/lib/request_log_analyzer/file_format.rb +46 -49
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
- data/lib/request_log_analyzer/file_format/apache.rb +42 -45
- data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
- data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
- data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
- data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
- data/lib/request_log_analyzer/file_format/merb.rb +13 -17
- data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
- data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
- data/lib/request_log_analyzer/file_format/oink.rb +30 -31
- data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
- data/lib/request_log_analyzer/file_format/rack.rb +0 -2
- data/lib/request_log_analyzer/file_format/rails.rb +100 -104
- data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
- data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
- data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
- data/lib/request_log_analyzer/filter.rb +0 -2
- data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
- data/lib/request_log_analyzer/filter/field.rb +3 -6
- data/lib/request_log_analyzer/filter/timespan.rb +2 -6
- data/lib/request_log_analyzer/line_definition.rb +16 -19
- data/lib/request_log_analyzer/log_processor.rb +10 -14
- data/lib/request_log_analyzer/mailer.rb +9 -12
- data/lib/request_log_analyzer/output.rb +12 -14
- data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
- data/lib/request_log_analyzer/output/html.rb +11 -14
- data/lib/request_log_analyzer/request.rb +53 -33
- data/lib/request_log_analyzer/source.rb +2 -5
- data/lib/request_log_analyzer/source/log_parser.rb +9 -16
- data/lib/request_log_analyzer/tracker.rb +10 -12
- data/lib/request_log_analyzer/tracker/duration.rb +4 -6
- data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
- data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
- data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
- data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
- data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +6 -6
- data/spec/integration/command_line_usage_spec.rb +33 -33
- data/spec/integration/mailer_spec.rb +181 -185
- data/spec/integration/munin_plugins_rails_spec.rb +20 -20
- data/spec/integration/scout_spec.rb +40 -41
- data/spec/lib/helpers.rb +8 -9
- data/spec/lib/macros.rb +2 -4
- data/spec/lib/matchers.rb +20 -25
- data/spec/lib/mocks.rb +10 -11
- data/spec/lib/testing_format.rb +8 -10
- data/spec/spec_helper.rb +5 -1
- data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
- data/spec/unit/aggregator/summarizer_spec.rb +7 -7
- data/spec/unit/controller/controller_spec.rb +14 -14
- data/spec/unit/controller/log_processor_spec.rb +3 -3
- data/spec/unit/database/base_class_spec.rb +36 -37
- data/spec/unit/database/connection_spec.rb +10 -10
- data/spec/unit/database/database_spec.rb +11 -11
- data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
- data/spec/unit/file_format/apache_format_spec.rb +57 -52
- data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
- data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
- data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
- data/spec/unit/file_format/file_format_api_spec.rb +19 -19
- data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
- data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
- data/spec/unit/file_format/inheritance_spec.rb +13 -0
- data/spec/unit/file_format/line_definition_spec.rb +35 -33
- data/spec/unit/file_format/merb_format_spec.rb +13 -11
- data/spec/unit/file_format/mysql_format_spec.rb +24 -24
- data/spec/unit/file_format/oink_format_spec.rb +29 -29
- data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
- data/spec/unit/file_format/rack_format_spec.rb +36 -31
- data/spec/unit/file_format/rails3_format_spec.rb +46 -46
- data/spec/unit/file_format/rails_format_spec.rb +52 -53
- data/spec/unit/file_format/w3c_format_spec.rb +27 -24
- data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
- data/spec/unit/filter/field_filter_spec.rb +26 -26
- data/spec/unit/filter/filter_spec.rb +4 -4
- data/spec/unit/filter/timespan_filter_spec.rb +22 -22
- data/spec/unit/mailer_spec.rb +21 -21
- data/spec/unit/request_spec.rb +29 -29
- data/spec/unit/source/log_parser_spec.rb +5 -5
- data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
- data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
- data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
- data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
- data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
- data/spec/unit/tracker/tracker_api_spec.rb +43 -44
- data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
- metadata +38 -35
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# FileFormat for Amazon S3 access logs.
|
|
4
3
|
#
|
|
5
4
|
# Access logs are disabled by default on Amazon S3. To enable logging, see
|
|
6
5
|
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
|
|
7
6
|
class AmazonS3 < Base
|
|
8
|
-
|
|
9
7
|
extend CommonRegularExpressions
|
|
10
8
|
|
|
11
9
|
line_definition :access do |line|
|
|
@@ -24,10 +22,10 @@ module RequestLogAnalyzer::FileFormat
|
|
|
24
22
|
line.capture(:request_uri)
|
|
25
23
|
line.capture(:http_status).as(:integer)
|
|
26
24
|
line.capture(:error_code).as(:nillable_string)
|
|
27
|
-
line.capture(:bytes_sent).as(:traffic, :
|
|
28
|
-
line.capture(:object_size).as(:traffic, :
|
|
29
|
-
line.capture(:total_time).as(:duration, :
|
|
30
|
-
line.capture(:turnaround_time).as(:duration, :
|
|
25
|
+
line.capture(:bytes_sent).as(:traffic, unit: :byte)
|
|
26
|
+
line.capture(:object_size).as(:traffic, unit: :byte)
|
|
27
|
+
line.capture(:total_time).as(:duration, unit: :msec)
|
|
28
|
+
line.capture(:turnaround_time).as(:duration, unit: :msec)
|
|
31
29
|
line.capture(:referer).as(:referer)
|
|
32
30
|
line.capture(:user_agent).as(:user_agent)
|
|
33
31
|
end
|
|
@@ -36,39 +34,37 @@ module RequestLogAnalyzer::FileFormat
|
|
|
36
34
|
analyze.timespan
|
|
37
35
|
analyze.hourly_spread
|
|
38
36
|
|
|
39
|
-
analyze.frequency :
|
|
40
|
-
analyze.duration :
|
|
41
|
-
analyze.traffic
|
|
42
|
-
analyze.frequency :
|
|
43
|
-
analyze.frequency :
|
|
37
|
+
analyze.frequency category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Most popular items'
|
|
38
|
+
analyze.duration duration: :total_time, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Request duration'
|
|
39
|
+
analyze.traffic traffic: :bytes_sent, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Traffic'
|
|
40
|
+
analyze.frequency category: :http_status, title: 'HTTP status codes'
|
|
41
|
+
analyze.frequency category: :error_code, title: 'Error codes'
|
|
44
42
|
end
|
|
45
43
|
|
|
46
44
|
class Request < RequestLogAnalyzer::Request
|
|
47
|
-
|
|
48
|
-
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
45
|
+
MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
49
46
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
50
47
|
|
|
51
48
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
52
49
|
# to speed up parsing.
|
|
53
|
-
def convert_timestamp(value,
|
|
54
|
-
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
50
|
+
def convert_timestamp(value, _definition)
|
|
51
|
+
"#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
|
|
55
52
|
end
|
|
56
53
|
|
|
57
54
|
# Make sure that the string '-' is parsed as a nil value.
|
|
58
|
-
def convert_nillable_string(value,
|
|
55
|
+
def convert_nillable_string(value, _definition)
|
|
59
56
|
value == '-' ? nil : value
|
|
60
57
|
end
|
|
61
58
|
|
|
62
59
|
# Can be implemented in subclasses for improved categorizations
|
|
63
|
-
def convert_referer(value,
|
|
60
|
+
def convert_referer(value, _definition)
|
|
64
61
|
value == '-' ? nil : value
|
|
65
62
|
end
|
|
66
63
|
|
|
67
64
|
# Can be implemented in subclasses for improved categorizations
|
|
68
|
-
def convert_user_agent(value,
|
|
65
|
+
def convert_user_agent(value, _definition)
|
|
69
66
|
value == '-' ? nil : value
|
|
70
67
|
end
|
|
71
68
|
end
|
|
72
|
-
|
|
73
69
|
end
|
|
74
70
|
end
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The Apache file format is able to log Apache access.log files.
|
|
4
3
|
#
|
|
5
4
|
# The access.log can be configured in Apache to have many different formats. In theory, this
|
|
@@ -13,18 +12,17 @@ module RequestLogAnalyzer::FileFormat
|
|
|
13
12
|
# From the command line, you can provide the format string using the <tt>--apache-format</tt>
|
|
14
13
|
# command line option.
|
|
15
14
|
class Apache < Base
|
|
16
|
-
|
|
17
15
|
extend CommonRegularExpressions
|
|
18
16
|
|
|
19
17
|
# A hash of predefined Apache log formats
|
|
20
18
|
LOG_FORMAT_DEFAULTS = {
|
|
21
|
-
:
|
|
22
|
-
:
|
|
23
|
-
:
|
|
24
|
-
:
|
|
25
|
-
:
|
|
26
|
-
:
|
|
27
|
-
:
|
|
19
|
+
common: '%h %l %u %t "%r" %>s %b',
|
|
20
|
+
combined: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
|
|
21
|
+
vhost_combined: '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
|
|
22
|
+
nginx: '%a %t %h %u "%r" %>s %b',
|
|
23
|
+
rack: '%h %l %u %t "%r" %>s %b %T',
|
|
24
|
+
referer: '%{Referer}i -> %U',
|
|
25
|
+
agent: '%{User-agent}i'
|
|
28
26
|
}
|
|
29
27
|
|
|
30
28
|
# I have encountered two timestamp types, with timezone and without. Parse both.
|
|
@@ -32,26 +30,26 @@ module RequestLogAnalyzer::FileFormat
|
|
|
32
30
|
|
|
33
31
|
# A hash that defines how the log format directives should be parsed.
|
|
34
32
|
LOG_DIRECTIVES = {
|
|
35
|
-
'%' => { nil => { :
|
|
36
|
-
'v' => { nil => { :
|
|
37
|
-
'h' => { nil => { :
|
|
38
|
-
'a' => { nil => { :
|
|
39
|
-
'b' => { nil => { :
|
|
40
|
-
'c' => { nil => { :
|
|
41
|
-
'D' => { nil => { :
|
|
42
|
-
'micro' => { :
|
|
43
|
-
'milli' => { :
|
|
33
|
+
'%' => { nil => { regexp: '%', captures: [] } },
|
|
34
|
+
'v' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :vhost, type: :string }] } },
|
|
35
|
+
'h' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :remote_host, type: :string }] } },
|
|
36
|
+
'a' => { nil => { regexp: "(#{ip_address})", captures: [{ name: :remote_ip, type: :string }] } },
|
|
37
|
+
'b' => { nil => { regexp: '(\d+|-)', captures: [{ name: :bytes_sent, type: :traffic }] } },
|
|
38
|
+
'c' => { nil => { regexp: '(\+|\-|\X)', captures: [{ name: :connection_status, type: :integer }] } },
|
|
39
|
+
'D' => { nil => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
|
|
40
|
+
'micro' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
|
|
41
|
+
'milli' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :msec }] }
|
|
44
42
|
},
|
|
45
|
-
'l' => { nil => { :
|
|
46
|
-
'T' => { nil => { :
|
|
47
|
-
't' => { nil => { :
|
|
48
|
-
's' => { nil => { :
|
|
49
|
-
'u' => { nil => { :
|
|
50
|
-
'U' => { nil => { :
|
|
51
|
-
'r' => { nil => { :
|
|
52
|
-
|
|
53
|
-
'i' => { 'Referer' => { :
|
|
54
|
-
'User-agent' => { :
|
|
43
|
+
'l' => { nil => { regexp: '([\w-]+)', captures: [{ name: :remote_logname, type: :nillable_string }] } },
|
|
44
|
+
'T' => { nil => { regexp: '(\d+(?:\.\d+)?|-)', captures: [{ name: :duration, type: :duration, unit: :sec }] } },
|
|
45
|
+
't' => { nil => { regexp: "\\[(#{APACHE_TIMESTAMP})?\\]", captures: [{ name: :timestamp, type: :timestamp }] } },
|
|
46
|
+
's' => { nil => { regexp: '(\d{3})', captures: [{ name: :http_status, type: :integer }] } },
|
|
47
|
+
'u' => { nil => { regexp: '(\/\S*|-)', captures: [{ name: :user, type: :nillable_string }] } },
|
|
48
|
+
'U' => { nil => { regexp: '(\/\S*)', captures: [{ name: :path, type: :string }] } },
|
|
49
|
+
'r' => { nil => { regexp: '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', captures: [{ name: :http_method, type: :string },
|
|
50
|
+
{ name: :path, type: :path }, { name: :http_version, type: :string }] } },
|
|
51
|
+
'i' => { 'Referer' => { regexp: '(\S+)', captures: [{ name: :referer, type: :nillable_string }] },
|
|
52
|
+
'User-agent' => { regexp: '(.*)', captures: [{ name: :user_agent, type: :user_agent }] }
|
|
55
53
|
}
|
|
56
54
|
}
|
|
57
55
|
|
|
@@ -61,7 +59,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
61
59
|
def self.create(*args)
|
|
62
60
|
access_line = access_line_definition(args.first)
|
|
63
61
|
trackers = report_trackers(access_line) + report_definer.trackers
|
|
64
|
-
|
|
62
|
+
new(line_definer.line_definitions.merge(access: access_line), trackers)
|
|
65
63
|
end
|
|
66
64
|
|
|
67
65
|
# Creates the access log line definition based on the Apache log format string
|
|
@@ -90,8 +88,8 @@ module RequestLogAnalyzer::FileFormat
|
|
|
90
88
|
end
|
|
91
89
|
|
|
92
90
|
# Return a new line definition object
|
|
93
|
-
|
|
94
|
-
:
|
|
91
|
+
RequestLogAnalyzer::LineDefinition.new(:access, regexp: Regexp.new(line_regexp),
|
|
92
|
+
captures: captures, header: true, footer: true)
|
|
95
93
|
end
|
|
96
94
|
|
|
97
95
|
# Sets up the report trackers according to the fields captured by the access line definition.
|
|
@@ -101,40 +99,39 @@ module RequestLogAnalyzer::FileFormat
|
|
|
101
99
|
analyze.timespan if line_definition.captures?(:timestamp)
|
|
102
100
|
analyze.hourly_spread if line_definition.captures?(:timestamp)
|
|
103
101
|
|
|
104
|
-
analyze.frequency :
|
|
105
|
-
analyze.frequency :
|
|
106
|
-
analyze.frequency :
|
|
102
|
+
analyze.frequency category: :http_method, title: 'HTTP methods' if line_definition.captures?(:http_method)
|
|
103
|
+
analyze.frequency category: :http_status, title: 'HTTP statuses' if line_definition.captures?(:http_status)
|
|
104
|
+
analyze.frequency category: lambda { |r| r.category }, title: 'Most popular URIs' if line_definition.captures?(:path)
|
|
107
105
|
|
|
108
|
-
analyze.frequency :
|
|
109
|
-
analyze.frequency :
|
|
106
|
+
analyze.frequency category: :user_agent, title: 'User agents' if line_definition.captures?(:user_agent)
|
|
107
|
+
analyze.frequency category: :referer, title: 'Referers' if line_definition.captures?(:referer)
|
|
110
108
|
|
|
111
|
-
analyze.duration :
|
|
112
|
-
analyze.traffic
|
|
109
|
+
analyze.duration duration: :duration, category: lambda { |r| r.category }, title: 'Request duration' if line_definition.captures?(:duration)
|
|
110
|
+
analyze.traffic traffic: :bytes_sent, category: lambda { |r| r.category }, title: 'Traffic' if line_definition.captures?(:bytes_sent)
|
|
113
111
|
|
|
114
|
-
|
|
112
|
+
analyze.trackers
|
|
115
113
|
end
|
|
116
114
|
|
|
117
115
|
# Define a custom Request class for the Apache file format to speed up timestamp handling.
|
|
118
116
|
class Request < RequestLogAnalyzer::Request
|
|
119
|
-
|
|
120
117
|
def category
|
|
121
118
|
first(:path)
|
|
122
119
|
end
|
|
123
120
|
|
|
124
|
-
MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
121
|
+
MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
|
|
125
122
|
'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
|
|
126
123
|
|
|
127
124
|
# Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
|
|
128
125
|
# to speed up parsing.
|
|
129
|
-
def convert_timestamp(value,
|
|
130
|
-
"#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
|
|
126
|
+
def convert_timestamp(value, _definition)
|
|
127
|
+
"#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
|
|
131
128
|
end
|
|
132
129
|
|
|
133
130
|
# This function can be overridden to simplify the user agent string for better
|
|
134
131
|
# categorization in the reports
|
|
135
|
-
def convert_user_agent(value,
|
|
132
|
+
def convert_user_agent(value, _definition)
|
|
136
133
|
value # TODO
|
|
137
134
|
end
|
|
138
135
|
end
|
|
139
136
|
end
|
|
140
|
-
end
|
|
137
|
+
end
|
|
@@ -1,40 +1,38 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The DelayedJob file format parsed log files that are created by DelayedJob.
|
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
4
|
class DelayedJob < Base
|
|
6
|
-
|
|
7
5
|
line_definition :job_lock do |line|
|
|
8
6
|
line.header = true
|
|
9
7
|
line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
|
|
10
|
-
|
|
8
|
+
|
|
11
9
|
line.capture(:job)
|
|
12
10
|
end
|
|
13
|
-
|
|
11
|
+
|
|
14
12
|
line_definition :job_completed do |line|
|
|
15
13
|
line.footer = true
|
|
16
14
|
line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
|
|
17
15
|
|
|
18
16
|
line.capture(:completed_job)
|
|
19
|
-
line.capture(:duration).as(:duration, :
|
|
17
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
|
20
18
|
end
|
|
21
|
-
|
|
19
|
+
|
|
22
20
|
line_definition :job_failed do |line|
|
|
23
21
|
line.footer = true
|
|
24
22
|
line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
|
|
25
|
-
|
|
23
|
+
|
|
26
24
|
line.capture(:failed_job)
|
|
27
25
|
line.capture(:exception)
|
|
28
26
|
line.capture(:attempts).as(:integer)
|
|
29
27
|
end
|
|
30
|
-
|
|
28
|
+
|
|
31
29
|
line_definition :job_lock_failed do |line|
|
|
32
30
|
line.footer = true
|
|
33
31
|
line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
|
|
34
|
-
|
|
32
|
+
|
|
35
33
|
line.capture(:locked_job)
|
|
36
34
|
end
|
|
37
|
-
|
|
35
|
+
|
|
38
36
|
# line_definition :batch_completed do |line|
|
|
39
37
|
# line.header = true
|
|
40
38
|
# line.footer = true
|
|
@@ -44,12 +42,12 @@ module RequestLogAnalyzer::FileFormat
|
|
|
44
42
|
# line.capture(:mean_duration).as(:duration, :unit => :sec)
|
|
45
43
|
# line.capture(:failed_amount).as(:integer)
|
|
46
44
|
# end
|
|
47
|
-
|
|
45
|
+
|
|
48
46
|
report do |analyze|
|
|
49
|
-
analyze.frequency :job, :
|
|
50
|
-
analyze.frequency :job, :
|
|
51
|
-
|
|
52
|
-
analyze.duration :duration, :
|
|
47
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
|
48
|
+
analyze.frequency :job, if: lambda { |request| request[:attempts] == 1 }, title: 'Failed jobs'
|
|
49
|
+
|
|
50
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
|
53
51
|
end
|
|
54
52
|
end
|
|
55
53
|
end
|
|
@@ -1,38 +1,36 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The DelayedJob2 file format parsed log files that are created by DelayedJob 2.0.
|
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
4
|
class DelayedJob2 < Base
|
|
6
|
-
|
|
7
5
|
extend CommonRegularExpressions
|
|
8
|
-
|
|
6
|
+
|
|
9
7
|
line_definition :job_lock do |line|
|
|
10
8
|
line.header = true
|
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
|
12
|
-
|
|
10
|
+
|
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
|
14
12
|
line.capture(:host)
|
|
15
13
|
line.capture(:pid).as(:integer)
|
|
16
14
|
line.capture(:job)
|
|
17
15
|
end
|
|
18
|
-
|
|
16
|
+
|
|
19
17
|
line_definition :job_completed do |line|
|
|
20
18
|
line.footer = true
|
|
21
19
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
|
|
22
20
|
line.capture(:timestamp).as(:timestamp)
|
|
23
21
|
line.capture(:host)
|
|
24
22
|
line.capture(:pid).as(:integer)
|
|
25
|
-
line.capture(:duration).as(:duration, :
|
|
23
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
|
26
24
|
end
|
|
27
|
-
|
|
25
|
+
|
|
28
26
|
report do |analyze|
|
|
29
27
|
analyze.timespan
|
|
30
28
|
analyze.hourly_spread
|
|
31
29
|
|
|
32
|
-
analyze.frequency :job, :
|
|
33
|
-
#analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
|
34
|
-
|
|
35
|
-
analyze.duration :duration, :
|
|
30
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
|
31
|
+
# analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
|
32
|
+
|
|
33
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
|
36
34
|
end
|
|
37
35
|
end
|
|
38
36
|
end
|
|
@@ -1,21 +1,19 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The DelayedJob21 file format parsed log files that are created by DelayedJob 2.1 or higher.
|
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
4
|
class DelayedJob21 < Base
|
|
6
|
-
|
|
7
5
|
extend CommonRegularExpressions
|
|
8
|
-
|
|
6
|
+
|
|
9
7
|
line_definition :job_lock do |line|
|
|
10
8
|
line.header = true
|
|
11
9
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
|
|
12
|
-
|
|
10
|
+
|
|
13
11
|
line.capture(:timestamp).as(:timestamp)
|
|
14
12
|
line.capture(:host)
|
|
15
13
|
line.capture(:pid).as(:integer)
|
|
16
14
|
line.capture(:job)
|
|
17
15
|
end
|
|
18
|
-
|
|
16
|
+
|
|
19
17
|
line_definition :job_completed do |line|
|
|
20
18
|
line.footer = true
|
|
21
19
|
line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
|
|
@@ -23,17 +21,17 @@ module RequestLogAnalyzer::FileFormat
|
|
|
23
21
|
line.capture(:host)
|
|
24
22
|
line.capture(:pid).as(:integer)
|
|
25
23
|
line.capture(:job)
|
|
26
|
-
line.capture(:duration).as(:duration, :
|
|
24
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
|
27
25
|
end
|
|
28
|
-
|
|
26
|
+
|
|
29
27
|
report do |analyze|
|
|
30
28
|
analyze.timespan
|
|
31
29
|
analyze.hourly_spread
|
|
32
30
|
|
|
33
|
-
analyze.frequency :job, :
|
|
34
|
-
#analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
|
35
|
-
|
|
36
|
-
analyze.duration :duration, :
|
|
31
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
|
32
|
+
# analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
|
33
|
+
|
|
34
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
|
37
35
|
end
|
|
38
36
|
end
|
|
39
37
|
end
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The DelayedJob3 file format parsed log files that are created by DelayedJob 3.0 or higher.
|
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
4
|
class DelayedJob3 < Base
|
|
6
|
-
|
|
7
5
|
extend CommonRegularExpressions
|
|
8
6
|
|
|
9
7
|
line_definition :job_completed do |line|
|
|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
14
12
|
line.capture(:host)
|
|
15
13
|
line.capture(:pid).as(:integer)
|
|
16
14
|
line.capture(:job)
|
|
17
|
-
line.capture(:duration).as(:duration, :
|
|
15
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
|
18
16
|
end
|
|
19
17
|
|
|
20
18
|
line_definition :job_failed do |line|
|
|
@@ -39,15 +37,14 @@ module RequestLogAnalyzer::FileFormat
|
|
|
39
37
|
line.capture(:failures).as(:integer)
|
|
40
38
|
end
|
|
41
39
|
|
|
42
|
-
|
|
43
40
|
report do |analyze|
|
|
44
41
|
analyze.timespan
|
|
45
42
|
analyze.hourly_spread
|
|
46
43
|
|
|
47
|
-
analyze.frequency :job, :
|
|
48
|
-
analyze.frequency :job, :
|
|
49
|
-
analyze.frequency :failures, :
|
|
50
|
-
analyze.duration :duration, :
|
|
44
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
|
45
|
+
analyze.frequency :job, category: :job, line_type: :job_failed, title: 'Failed jobs'
|
|
46
|
+
analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
|
|
47
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
|
51
48
|
end
|
|
52
49
|
end
|
|
53
50
|
end
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
module RequestLogAnalyzer::FileFormat
|
|
2
|
-
|
|
3
2
|
# The DelayedJob4 file format parsed log files that are created by DelayedJob 4.0 or higher.
|
|
4
3
|
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
4
|
class DelayedJob4 < Base
|
|
6
|
-
|
|
7
5
|
extend CommonRegularExpressions
|
|
8
6
|
|
|
9
7
|
line_definition :job_completed do |line|
|
|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
|
|
|
14
12
|
line.capture(:host)
|
|
15
13
|
line.capture(:pid).as(:integer)
|
|
16
14
|
line.capture(:job)
|
|
17
|
-
line.capture(:duration).as(:duration, :
|
|
15
|
+
line.capture(:duration).as(:duration, unit: :sec)
|
|
18
16
|
end
|
|
19
17
|
|
|
20
18
|
line_definition :job_failed do |line|
|
|
@@ -40,15 +38,14 @@ module RequestLogAnalyzer::FileFormat
|
|
|
40
38
|
line.capture(:failures).as(:integer)
|
|
41
39
|
end
|
|
42
40
|
|
|
43
|
-
|
|
44
41
|
report do |analyze|
|
|
45
42
|
analyze.timespan
|
|
46
43
|
analyze.hourly_spread
|
|
47
44
|
|
|
48
|
-
analyze.frequency :job, :
|
|
49
|
-
analyze.frequency :job, :
|
|
50
|
-
analyze.frequency :failures, :
|
|
51
|
-
analyze.duration :duration, :
|
|
45
|
+
analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
|
|
46
|
+
analyze.frequency :job, category: lambda { |r| "#{r[:job]} #{r[:error]}" }, line_type: :job_failed, title: 'Failed jobs'
|
|
47
|
+
analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
|
|
48
|
+
analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
|
|
52
49
|
end
|
|
53
50
|
end
|
|
54
51
|
end
|