request-log-analyzer 1.13.1 → 1.13.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/bin/console +17 -0
  4. data/lib/cli/command_line_arguments.rb +29 -36
  5. data/lib/cli/database_console.rb +1 -3
  6. data/lib/cli/database_console_init.rb +11 -11
  7. data/lib/cli/progressbar.rb +30 -32
  8. data/lib/cli/tools.rb +20 -23
  9. data/lib/request_log_analyzer.rb +8 -8
  10. data/lib/request_log_analyzer/aggregator.rb +4 -7
  11. data/lib/request_log_analyzer/aggregator/database_inserter.rb +10 -13
  12. data/lib/request_log_analyzer/aggregator/echo.rb +5 -7
  13. data/lib/request_log_analyzer/aggregator/summarizer.rb +15 -18
  14. data/lib/request_log_analyzer/class_level_inheritable_attributes.rb +23 -0
  15. data/lib/request_log_analyzer/controller.rb +36 -42
  16. data/lib/request_log_analyzer/database.rb +4 -6
  17. data/lib/request_log_analyzer/database/base.rb +39 -41
  18. data/lib/request_log_analyzer/database/connection.rb +8 -10
  19. data/lib/request_log_analyzer/database/request.rb +1 -3
  20. data/lib/request_log_analyzer/database/source.rb +0 -2
  21. data/lib/request_log_analyzer/database/warning.rb +4 -6
  22. data/lib/request_log_analyzer/file_format.rb +46 -49
  23. data/lib/request_log_analyzer/file_format/amazon_s3.rb +15 -19
  24. data/lib/request_log_analyzer/file_format/apache.rb +42 -45
  25. data/lib/request_log_analyzer/file_format/delayed_job.rb +13 -15
  26. data/lib/request_log_analyzer/file_format/delayed_job2.rb +9 -11
  27. data/lib/request_log_analyzer/file_format/delayed_job21.rb +9 -11
  28. data/lib/request_log_analyzer/file_format/delayed_job3.rb +5 -8
  29. data/lib/request_log_analyzer/file_format/delayed_job4.rb +5 -8
  30. data/lib/request_log_analyzer/file_format/haproxy.rb +44 -48
  31. data/lib/request_log_analyzer/file_format/merb.rb +13 -17
  32. data/lib/request_log_analyzer/file_format/mysql.rb +21 -25
  33. data/lib/request_log_analyzer/file_format/nginx.rb +0 -2
  34. data/lib/request_log_analyzer/file_format/oink.rb +30 -31
  35. data/lib/request_log_analyzer/file_format/postgresql.rb +11 -15
  36. data/lib/request_log_analyzer/file_format/rack.rb +0 -2
  37. data/lib/request_log_analyzer/file_format/rails.rb +100 -104
  38. data/lib/request_log_analyzer/file_format/rails3.rb +19 -23
  39. data/lib/request_log_analyzer/file_format/rails_development.rb +0 -1
  40. data/lib/request_log_analyzer/file_format/w3c.rb +16 -18
  41. data/lib/request_log_analyzer/filter.rb +0 -2
  42. data/lib/request_log_analyzer/filter/anonymize.rb +4 -7
  43. data/lib/request_log_analyzer/filter/field.rb +3 -6
  44. data/lib/request_log_analyzer/filter/timespan.rb +2 -6
  45. data/lib/request_log_analyzer/line_definition.rb +16 -19
  46. data/lib/request_log_analyzer/log_processor.rb +10 -14
  47. data/lib/request_log_analyzer/mailer.rb +9 -12
  48. data/lib/request_log_analyzer/output.rb +12 -14
  49. data/lib/request_log_analyzer/output/fixed_width.rb +21 -28
  50. data/lib/request_log_analyzer/output/html.rb +11 -14
  51. data/lib/request_log_analyzer/request.rb +53 -33
  52. data/lib/request_log_analyzer/source.rb +2 -5
  53. data/lib/request_log_analyzer/source/log_parser.rb +9 -16
  54. data/lib/request_log_analyzer/tracker.rb +10 -12
  55. data/lib/request_log_analyzer/tracker/duration.rb +4 -6
  56. data/lib/request_log_analyzer/tracker/frequency.rb +9 -11
  57. data/lib/request_log_analyzer/tracker/hourly_spread.rb +8 -11
  58. data/lib/request_log_analyzer/tracker/numeric_value.rb +40 -44
  59. data/lib/request_log_analyzer/tracker/timespan.rb +5 -8
  60. data/lib/request_log_analyzer/tracker/traffic.rb +8 -10
  61. data/lib/request_log_analyzer/version.rb +1 -1
  62. data/request-log-analyzer.gemspec +6 -6
  63. data/spec/integration/command_line_usage_spec.rb +33 -33
  64. data/spec/integration/mailer_spec.rb +181 -185
  65. data/spec/integration/munin_plugins_rails_spec.rb +20 -20
  66. data/spec/integration/scout_spec.rb +40 -41
  67. data/spec/lib/helpers.rb +8 -9
  68. data/spec/lib/macros.rb +2 -4
  69. data/spec/lib/matchers.rb +20 -25
  70. data/spec/lib/mocks.rb +10 -11
  71. data/spec/lib/testing_format.rb +8 -10
  72. data/spec/spec_helper.rb +5 -1
  73. data/spec/unit/aggregator/database_inserter_spec.rb +23 -23
  74. data/spec/unit/aggregator/summarizer_spec.rb +7 -7
  75. data/spec/unit/controller/controller_spec.rb +14 -14
  76. data/spec/unit/controller/log_processor_spec.rb +3 -3
  77. data/spec/unit/database/base_class_spec.rb +36 -37
  78. data/spec/unit/database/connection_spec.rb +10 -10
  79. data/spec/unit/database/database_spec.rb +11 -11
  80. data/spec/unit/file_format/amazon_s3_format_spec.rb +66 -62
  81. data/spec/unit/file_format/apache_format_spec.rb +57 -52
  82. data/spec/unit/file_format/common_regular_expressions_spec.rb +18 -21
  83. data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -16
  84. data/spec/unit/file_format/delayed_job2_format_spec.rb +22 -16
  85. data/spec/unit/file_format/delayed_job3_format_spec.rb +14 -10
  86. data/spec/unit/file_format/delayed_job4_format_spec.rb +14 -10
  87. data/spec/unit/file_format/delayed_job_format_spec.rb +12 -12
  88. data/spec/unit/file_format/file_format_api_spec.rb +19 -19
  89. data/spec/unit/file_format/format_autodetection_spec.rb +7 -7
  90. data/spec/unit/file_format/haproxy_format_spec.rb +53 -49
  91. data/spec/unit/file_format/inheritance_spec.rb +13 -0
  92. data/spec/unit/file_format/line_definition_spec.rb +35 -33
  93. data/spec/unit/file_format/merb_format_spec.rb +13 -11
  94. data/spec/unit/file_format/mysql_format_spec.rb +24 -24
  95. data/spec/unit/file_format/oink_format_spec.rb +29 -29
  96. data/spec/unit/file_format/postgresql_format_spec.rb +9 -9
  97. data/spec/unit/file_format/rack_format_spec.rb +36 -31
  98. data/spec/unit/file_format/rails3_format_spec.rb +46 -46
  99. data/spec/unit/file_format/rails_format_spec.rb +52 -53
  100. data/spec/unit/file_format/w3c_format_spec.rb +27 -24
  101. data/spec/unit/filter/anonymize_filter_spec.rb +7 -7
  102. data/spec/unit/filter/field_filter_spec.rb +26 -26
  103. data/spec/unit/filter/filter_spec.rb +4 -4
  104. data/spec/unit/filter/timespan_filter_spec.rb +22 -22
  105. data/spec/unit/mailer_spec.rb +21 -21
  106. data/spec/unit/request_spec.rb +29 -29
  107. data/spec/unit/source/log_parser_spec.rb +5 -5
  108. data/spec/unit/tracker/duration_tracker_spec.rb +23 -23
  109. data/spec/unit/tracker/frequency_tracker_spec.rb +29 -30
  110. data/spec/unit/tracker/hourly_spread_spec.rb +35 -35
  111. data/spec/unit/tracker/numeric_value_tracker_spec.rb +71 -72
  112. data/spec/unit/tracker/timespan_tracker_spec.rb +31 -31
  113. data/spec/unit/tracker/tracker_api_spec.rb +43 -44
  114. data/spec/unit/tracker/traffic_tracker_spec.rb +7 -7
  115. metadata +38 -35
@@ -1,11 +1,9 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # FileFormat for Amazon S3 access logs.
4
3
  #
5
4
  # Access logs are disabled by default on Amazon S3. To enable logging, see
6
5
  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
7
6
  class AmazonS3 < Base
8
-
9
7
  extend CommonRegularExpressions
10
8
 
11
9
  line_definition :access do |line|
@@ -24,10 +22,10 @@ module RequestLogAnalyzer::FileFormat
24
22
  line.capture(:request_uri)
25
23
  line.capture(:http_status).as(:integer)
26
24
  line.capture(:error_code).as(:nillable_string)
27
- line.capture(:bytes_sent).as(:traffic, :unit => :byte)
28
- line.capture(:object_size).as(:traffic, :unit => :byte)
29
- line.capture(:total_time).as(:duration, :unit => :msec)
30
- line.capture(:turnaround_time).as(:duration, :unit => :msec)
25
+ line.capture(:bytes_sent).as(:traffic, unit: :byte)
26
+ line.capture(:object_size).as(:traffic, unit: :byte)
27
+ line.capture(:total_time).as(:duration, unit: :msec)
28
+ line.capture(:turnaround_time).as(:duration, unit: :msec)
31
29
  line.capture(:referer).as(:referer)
32
30
  line.capture(:user_agent).as(:user_agent)
33
31
  end
@@ -36,39 +34,37 @@ module RequestLogAnalyzer::FileFormat
36
34
  analyze.timespan
37
35
  analyze.hourly_spread
38
36
 
39
- analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
40
- analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
41
- analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
42
- analyze.frequency :category => :http_status, :title => 'HTTP status codes'
43
- analyze.frequency :category => :error_code, :title => 'Error codes'
37
+ analyze.frequency category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Most popular items'
38
+ analyze.duration duration: :total_time, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Request duration'
39
+ analyze.traffic traffic: :bytes_sent, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Traffic'
40
+ analyze.frequency category: :http_status, title: 'HTTP status codes'
41
+ analyze.frequency category: :error_code, title: 'Error codes'
44
42
  end
45
43
 
46
44
  class Request < RequestLogAnalyzer::Request
47
-
48
- MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
45
+ MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
49
46
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
50
47
 
51
48
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
52
49
  # to speed up parsing.
53
- def convert_timestamp(value, definition)
54
- "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
50
+ def convert_timestamp(value, _definition)
51
+ "#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
55
52
  end
56
53
 
57
54
  # Make sure that the string '-' is parsed as a nil value.
58
- def convert_nillable_string(value, definition)
55
+ def convert_nillable_string(value, _definition)
59
56
  value == '-' ? nil : value
60
57
  end
61
58
 
62
59
  # Can be implemented in subclasses for improved categorizations
63
- def convert_referer(value, definition)
60
+ def convert_referer(value, _definition)
64
61
  value == '-' ? nil : value
65
62
  end
66
63
 
67
64
  # Can be implemented in subclasses for improved categorizations
68
- def convert_user_agent(value, definition)
65
+ def convert_user_agent(value, _definition)
69
66
  value == '-' ? nil : value
70
67
  end
71
68
  end
72
-
73
69
  end
74
70
  end
@@ -1,5 +1,4 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The Apache file format is able to log Apache access.log files.
4
3
  #
5
4
  # The access.log can be configured in Apache to have many different formats. In theory, this
@@ -13,18 +12,17 @@ module RequestLogAnalyzer::FileFormat
13
12
  # From the command line, you can provide the format string using the <tt>--apache-format</tt>
14
13
  # command line option.
15
14
  class Apache < Base
16
-
17
15
  extend CommonRegularExpressions
18
16
 
19
17
  # A hash of predefined Apache log formats
20
18
  LOG_FORMAT_DEFAULTS = {
21
- :common => '%h %l %u %t "%r" %>s %b',
22
- :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
23
- :vhost_combined => '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
24
- :nginx => '%a %t %h %u "%r" %>s %b',
25
- :rack => '%h %l %u %t "%r" %>s %b %T',
26
- :referer => '%{Referer}i -> %U',
27
- :agent => '%{User-agent}i'
19
+ common: '%h %l %u %t "%r" %>s %b',
20
+ combined: '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
21
+ vhost_combined: '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
22
+ nginx: '%a %t %h %u "%r" %>s %b',
23
+ rack: '%h %l %u %t "%r" %>s %b %T',
24
+ referer: '%{Referer}i -> %U',
25
+ agent: '%{User-agent}i'
28
26
  }
29
27
 
30
28
  # I have encountered two timestamp types, with timezone and without. Parse both.
@@ -32,26 +30,26 @@ module RequestLogAnalyzer::FileFormat
32
30
 
33
31
  # A hash that defines how the log format directives should be parsed.
34
32
  LOG_DIRECTIVES = {
35
- '%' => { nil => { :regexp => '%', :captures => [] } },
36
- 'v' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :vhost, :type => :string}] } },
37
- 'h' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :remote_host, :type => :string}] } },
38
- 'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
39
- 'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
40
- 'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },
41
- 'D' => { nil => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
42
- 'micro' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
43
- 'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
33
+ '%' => { nil => { regexp: '%', captures: [] } },
34
+ 'v' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :vhost, type: :string }] } },
35
+ 'h' => { nil => { regexp: "(#{hostname_or_ip_address})", captures: [{ name: :remote_host, type: :string }] } },
36
+ 'a' => { nil => { regexp: "(#{ip_address})", captures: [{ name: :remote_ip, type: :string }] } },
37
+ 'b' => { nil => { regexp: '(\d+|-)', captures: [{ name: :bytes_sent, type: :traffic }] } },
38
+ 'c' => { nil => { regexp: '(\+|\-|\X)', captures: [{ name: :connection_status, type: :integer }] } },
39
+ 'D' => { nil => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
40
+ 'micro' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :musec }] },
41
+ 'milli' => { regexp: '(\d+|-)', captures: [{ name: :duration, type: :duration, unit: :msec }] }
44
42
  },
45
- 'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
46
- 'T' => { nil => { :regexp => '(\d+(?:\.\d+)?|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
47
- 't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
48
- 's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
49
- 'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
50
- 'U' => { nil => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] } },
51
- 'r' => { nil => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
52
- {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]} },
53
- 'i' => { 'Referer' => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
54
- 'User-agent' => { :regexp => '(.*)', :captures => [{:name => :user_agent, :type => :user_agent}] }
43
+ 'l' => { nil => { regexp: '([\w-]+)', captures: [{ name: :remote_logname, type: :nillable_string }] } },
44
+ 'T' => { nil => { regexp: '(\d+(?:\.\d+)?|-)', captures: [{ name: :duration, type: :duration, unit: :sec }] } },
45
+ 't' => { nil => { regexp: "\\[(#{APACHE_TIMESTAMP})?\\]", captures: [{ name: :timestamp, type: :timestamp }] } },
46
+ 's' => { nil => { regexp: '(\d{3})', captures: [{ name: :http_status, type: :integer }] } },
47
+ 'u' => { nil => { regexp: '(\/\S*|-)', captures: [{ name: :user, type: :nillable_string }] } },
48
+ 'U' => { nil => { regexp: '(\/\S*)', captures: [{ name: :path, type: :string }] } },
49
+ 'r' => { nil => { regexp: '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', captures: [{ name: :http_method, type: :string },
50
+ { name: :path, type: :path }, { name: :http_version, type: :string }] } },
51
+ 'i' => { 'Referer' => { regexp: '(\S+)', captures: [{ name: :referer, type: :nillable_string }] },
52
+ 'User-agent' => { regexp: '(.*)', captures: [{ name: :user_agent, type: :user_agent }] }
55
53
  }
56
54
  }
57
55
 
@@ -61,7 +59,7 @@ module RequestLogAnalyzer::FileFormat
61
59
  def self.create(*args)
62
60
  access_line = access_line_definition(args.first)
63
61
  trackers = report_trackers(access_line) + report_definer.trackers
64
- self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
62
+ new(line_definer.line_definitions.merge(access: access_line), trackers)
65
63
  end
66
64
 
67
65
  # Creates the access log line definition based on the Apache log format string
@@ -90,8 +88,8 @@ module RequestLogAnalyzer::FileFormat
90
88
  end
91
89
 
92
90
  # Return a new line definition object
93
- return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
94
- :captures => captures, :header => true, :footer => true)
91
+ RequestLogAnalyzer::LineDefinition.new(:access, regexp: Regexp.new(line_regexp),
92
+ captures: captures, header: true, footer: true)
95
93
  end
96
94
 
97
95
  # Sets up the report trackers according to the fields captured by the access line definition.
@@ -101,40 +99,39 @@ module RequestLogAnalyzer::FileFormat
101
99
  analyze.timespan if line_definition.captures?(:timestamp)
102
100
  analyze.hourly_spread if line_definition.captures?(:timestamp)
103
101
 
104
- analyze.frequency :category => :http_method, :title => "HTTP methods" if line_definition.captures?(:http_method)
105
- analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
106
- analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs" if line_definition.captures?(:path)
102
+ analyze.frequency category: :http_method, title: 'HTTP methods' if line_definition.captures?(:http_method)
103
+ analyze.frequency category: :http_status, title: 'HTTP statuses' if line_definition.captures?(:http_status)
104
+ analyze.frequency category: lambda { |r| r.category }, title: 'Most popular URIs' if line_definition.captures?(:path)
107
105
 
108
- analyze.frequency :category => :user_agent, :title => "User agents" if line_definition.captures?(:user_agent)
109
- analyze.frequency :category => :referer, :title => "Referers" if line_definition.captures?(:referer)
106
+ analyze.frequency category: :user_agent, title: 'User agents' if line_definition.captures?(:user_agent)
107
+ analyze.frequency category: :referer, title: 'Referers' if line_definition.captures?(:referer)
110
108
 
111
- analyze.duration :duration => :duration, :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
112
- analyze.traffic :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic' if line_definition.captures?(:bytes_sent)
109
+ analyze.duration duration: :duration, category: lambda { |r| r.category }, title: 'Request duration' if line_definition.captures?(:duration)
110
+ analyze.traffic traffic: :bytes_sent, category: lambda { |r| r.category }, title: 'Traffic' if line_definition.captures?(:bytes_sent)
113
111
 
114
- return analyze.trackers
112
+ analyze.trackers
115
113
  end
116
114
 
117
115
  # Define a custom Request class for the Apache file format to speed up timestamp handling.
118
116
  class Request < RequestLogAnalyzer::Request
119
-
120
117
  def category
121
118
  first(:path)
122
119
  end
123
120
 
124
- MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
121
+ MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
125
122
  'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
126
123
 
127
124
  # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
128
125
  # to speed up parsing.
129
- def convert_timestamp(value, definition)
130
- "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
126
+ def convert_timestamp(value, _definition)
127
+ "#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i
131
128
  end
132
129
 
133
130
  # This function can be overridden to simplify the user agent string for better
134
131
  # categorization in the reports
135
- def convert_user_agent(value, definition)
132
+ def convert_user_agent(value, _definition)
136
133
  value # TODO
137
134
  end
138
135
  end
139
136
  end
140
- end
137
+ end
@@ -1,40 +1,38 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The DelayedJob file format parsed log files that are created by DelayedJob.
4
3
  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
5
4
  class DelayedJob < Base
6
-
7
5
  line_definition :job_lock do |line|
8
6
  line.header = true
9
7
  line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
10
-
8
+
11
9
  line.capture(:job)
12
10
  end
13
-
11
+
14
12
  line_definition :job_completed do |line|
15
13
  line.footer = true
16
14
  line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
17
15
 
18
16
  line.capture(:completed_job)
19
- line.capture(:duration).as(:duration, :unit => :sec)
17
+ line.capture(:duration).as(:duration, unit: :sec)
20
18
  end
21
-
19
+
22
20
  line_definition :job_failed do |line|
23
21
  line.footer = true
24
22
  line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
25
-
23
+
26
24
  line.capture(:failed_job)
27
25
  line.capture(:exception)
28
26
  line.capture(:attempts).as(:integer)
29
27
  end
30
-
28
+
31
29
  line_definition :job_lock_failed do |line|
32
30
  line.footer = true
33
31
  line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
34
-
32
+
35
33
  line.capture(:locked_job)
36
34
  end
37
-
35
+
38
36
  # line_definition :batch_completed do |line|
39
37
  # line.header = true
40
38
  # line.footer = true
@@ -44,12 +42,12 @@ module RequestLogAnalyzer::FileFormat
44
42
  # line.capture(:mean_duration).as(:duration, :unit => :sec)
45
43
  # line.capture(:failed_amount).as(:integer)
46
44
  # end
47
-
45
+
48
46
  report do |analyze|
49
- analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
50
- analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
51
-
52
- analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
47
+ analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
48
+ analyze.frequency :job, if: lambda { |request| request[:attempts] == 1 }, title: 'Failed jobs'
49
+
50
+ analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
53
51
  end
54
52
  end
55
53
  end
@@ -1,38 +1,36 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The DelayedJob2 file format parsed log files that are created by DelayedJob 2.0.
4
3
  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
5
4
  class DelayedJob2 < Base
6
-
7
5
  extend CommonRegularExpressions
8
-
6
+
9
7
  line_definition :job_lock do |line|
10
8
  line.header = true
11
9
  line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
12
-
10
+
13
11
  line.capture(:timestamp).as(:timestamp)
14
12
  line.capture(:host)
15
13
  line.capture(:pid).as(:integer)
16
14
  line.capture(:job)
17
15
  end
18
-
16
+
19
17
  line_definition :job_completed do |line|
20
18
  line.footer = true
21
19
  line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
22
20
  line.capture(:timestamp).as(:timestamp)
23
21
  line.capture(:host)
24
22
  line.capture(:pid).as(:integer)
25
- line.capture(:duration).as(:duration, :unit => :sec)
23
+ line.capture(:duration).as(:duration, unit: :sec)
26
24
  end
27
-
25
+
28
26
  report do |analyze|
29
27
  analyze.timespan
30
28
  analyze.hourly_spread
31
29
 
32
- analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
33
- #analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
34
-
35
- analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
30
+ analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
31
+ # analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
32
+
33
+ analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
36
34
  end
37
35
  end
38
36
  end
@@ -1,21 +1,19 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The DelayedJob21 file format parsed log files that are created by DelayedJob 2.1 or higher.
4
3
  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
5
4
  class DelayedJob21 < Base
6
-
7
5
  extend CommonRegularExpressions
8
-
6
+
9
7
  line_definition :job_lock do |line|
10
8
  line.header = true
11
9
  line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
12
-
10
+
13
11
  line.capture(:timestamp).as(:timestamp)
14
12
  line.capture(:host)
15
13
  line.capture(:pid).as(:integer)
16
14
  line.capture(:job)
17
15
  end
18
-
16
+
19
17
  line_definition :job_completed do |line|
20
18
  line.footer = true
21
19
  line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
@@ -23,17 +21,17 @@ module RequestLogAnalyzer::FileFormat
23
21
  line.capture(:host)
24
22
  line.capture(:pid).as(:integer)
25
23
  line.capture(:job)
26
- line.capture(:duration).as(:duration, :unit => :sec)
24
+ line.capture(:duration).as(:duration, unit: :sec)
27
25
  end
28
-
26
+
29
27
  report do |analyze|
30
28
  analyze.timespan
31
29
  analyze.hourly_spread
32
30
 
33
- analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
34
- #analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
35
-
36
- analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
31
+ analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
32
+ # analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
33
+
34
+ analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
37
35
  end
38
36
  end
39
37
  end
@@ -1,9 +1,7 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The DelayedJob3 file format parsed log files that are created by DelayedJob 3.0 or higher.
4
3
  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
5
4
  class DelayedJob3 < Base
6
-
7
5
  extend CommonRegularExpressions
8
6
 
9
7
  line_definition :job_completed do |line|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
14
12
  line.capture(:host)
15
13
  line.capture(:pid).as(:integer)
16
14
  line.capture(:job)
17
- line.capture(:duration).as(:duration, :unit => :sec)
15
+ line.capture(:duration).as(:duration, unit: :sec)
18
16
  end
19
17
 
20
18
  line_definition :job_failed do |line|
@@ -39,15 +37,14 @@ module RequestLogAnalyzer::FileFormat
39
37
  line.capture(:failures).as(:integer)
40
38
  end
41
39
 
42
-
43
40
  report do |analyze|
44
41
  analyze.timespan
45
42
  analyze.hourly_spread
46
43
 
47
- analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
48
- analyze.frequency :job, :category => :job, :line_type => :job_failed, :title => "Failed jobs"
49
- analyze.frequency :failures, :category => :job, :line_type => :job_deleted, :title => "Deleted jobs"
50
- analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
44
+ analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
45
+ analyze.frequency :job, category: :job, line_type: :job_failed, title: 'Failed jobs'
46
+ analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
47
+ analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
51
48
  end
52
49
  end
53
50
  end
@@ -1,9 +1,7 @@
1
1
  module RequestLogAnalyzer::FileFormat
2
-
3
2
  # The DelayedJob4 file format parsed log files that are created by DelayedJob 4.0 or higher.
4
3
  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
5
4
  class DelayedJob4 < Base
6
-
7
5
  extend CommonRegularExpressions
8
6
 
9
7
  line_definition :job_completed do |line|
@@ -14,7 +12,7 @@ module RequestLogAnalyzer::FileFormat
14
12
  line.capture(:host)
15
13
  line.capture(:pid).as(:integer)
16
14
  line.capture(:job)
17
- line.capture(:duration).as(:duration, :unit => :sec)
15
+ line.capture(:duration).as(:duration, unit: :sec)
18
16
  end
19
17
 
20
18
  line_definition :job_failed do |line|
@@ -40,15 +38,14 @@ module RequestLogAnalyzer::FileFormat
40
38
  line.capture(:failures).as(:integer)
41
39
  end
42
40
 
43
-
44
41
  report do |analyze|
45
42
  analyze.timespan
46
43
  analyze.hourly_spread
47
44
 
48
- analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
49
- analyze.frequency :job, :category => lambda { |r| "#{r[:job]} #{r[:error]}" }, :line_type => :job_failed, :title => "Failed jobs"
50
- analyze.frequency :failures, :category => :job, :line_type => :job_deleted, :title => "Deleted jobs"
51
- analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
45
+ analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs'
46
+ analyze.frequency :job, category: lambda { |r| "#{r[:job]} #{r[:error]}" }, line_type: :job_failed, title: 'Failed jobs'
47
+ analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs'
48
+ analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration'
52
49
  end
53
50
  end
54
51
  end