request-log-analyzer 1.5.3 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/request_log_analyzer/database.rb +1 -1
- data/lib/request_log_analyzer/file_format/delayed_job.rb +51 -0
- data/lib/request_log_analyzer/file_format/postgresql.rb +66 -0
- data/lib/request_log_analyzer/file_format/rails.rb +2 -2
- data/lib/request_log_analyzer/output/fixed_width.rb +1 -1
- data/lib/request_log_analyzer.rb +1 -1
- data/request-log-analyzer.gemspec +4 -4
- data/spec/fixtures/postgresql.log +2980 -0
- data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
- data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
- data/spec/unit/file_format/rails_format_spec.rb +8 -2
- data/tasks/github-gem.rake +6 -1
- metadata +14 -7
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
module RequestLogAnalyzer::FileFormat
|
|
2
|
+
|
|
3
|
+
# The DelayedJob file format parsed log files that are created by DelayedJob.
|
|
4
|
+
# By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
|
|
5
|
+
class DelayedJob < Base
|
|
6
|
+
|
|
7
|
+
line_definition :job_lock do |line|
|
|
8
|
+
line.header = true
|
|
9
|
+
line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
|
|
10
|
+
line.captures << { :name => :job, :type => :string }
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
line_definition :job_completed do |line|
|
|
14
|
+
line.footer = true
|
|
15
|
+
line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
|
|
16
|
+
line.captures << { :name => :completed_job, :type => :string } <<
|
|
17
|
+
{ :name => :duration, :type => :duration, :unit => :sec }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
line_definition :job_failed do |line|
|
|
21
|
+
line.footer = true
|
|
22
|
+
line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
|
|
23
|
+
line.captures << { :name => :failed_job, :type => :string } <<
|
|
24
|
+
{ :name => :exception, :type => :string } <<
|
|
25
|
+
{ :name => :attempts, :type => :integer }
|
|
26
|
+
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
line_definition :job_lock_failed do |line|
|
|
30
|
+
line.footer = true
|
|
31
|
+
line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
|
|
32
|
+
line.captures << { :name => :locked_job, :type => :string }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# line_definition :batch_completed do |line|
|
|
36
|
+
# line.header = true
|
|
37
|
+
# line.footer = true
|
|
38
|
+
# line.regexp = /(\d+) jobs processed at (\d+\.\d+) j\/s, (\d+) failed .../
|
|
39
|
+
# line.captures << { :name => :total_amount, :type => :integer } <<
|
|
40
|
+
# { :name => :mean_duration, :type => :duration, :unit => :sec } <<
|
|
41
|
+
# { :name => :failed_amount, :type => :integer }
|
|
42
|
+
# end
|
|
43
|
+
|
|
44
|
+
report do |analyze|
|
|
45
|
+
analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
|
|
46
|
+
analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs"
|
|
47
|
+
|
|
48
|
+
analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
module RequestLogAnalyzer::FileFormat
|
|
2
|
+
|
|
3
|
+
class Postgresql < Base
|
|
4
|
+
|
|
5
|
+
extend CommonRegularExpressions
|
|
6
|
+
|
|
7
|
+
line_definition :query do |line|
|
|
8
|
+
line.header = true
|
|
9
|
+
line.teaser = /LOG\: query\:/
|
|
10
|
+
line.regexp = /(#{timestamp('%y-%m-%d %k:%M:%S')})\ LOG: query:\s+(.*)/
|
|
11
|
+
line.captures << { :name => :timestamp, :type => :timestamp } <<
|
|
12
|
+
{ :name => :query_fragment, :type => :string }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
line_definition :duration do |line|
|
|
16
|
+
line.footer = true
|
|
17
|
+
line.teaser = /duration:/
|
|
18
|
+
line.regexp = /#{timestamp('%y-%m-%d %k:%M:%S')}\ LOG\: duration\: (.*)(\ )sec/
|
|
19
|
+
line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
|
|
20
|
+
{ :name => :query, :type => :sql } # Hack to gather up fragments
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
line_definition :query_fragment do |line|
|
|
24
|
+
line.regexp = /^(?!.*LOG)\s*(.*)\s*/
|
|
25
|
+
line.captures << { :name => :query_fragment, :type => :string }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
report do |analyze|
|
|
29
|
+
analyze.timespan
|
|
30
|
+
analyze.hourly_spread
|
|
31
|
+
analyze.duration :query_time, :category => :query, :title => 'Query time'
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
class Request < RequestLogAnalyzer::Request
|
|
35
|
+
|
|
36
|
+
def convert_sql(value, definition)
|
|
37
|
+
|
|
38
|
+
# Recreate the full SQL query by joining all the previous parts and this last line
|
|
39
|
+
sql = every(:query_fragment).join("\n") + value
|
|
40
|
+
|
|
41
|
+
# Sanitize an SQL query so that it can be used as a category field.
|
|
42
|
+
# sql.gsub!(/\/\*.*\*\//, '') # remove comments
|
|
43
|
+
sql.gsub!(/\s+/, ' ') # remove excessive whitespace
|
|
44
|
+
sql.gsub!(/"([^"]+)"/, '\1') # remove quotes from field names
|
|
45
|
+
sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date') # replace dates
|
|
46
|
+
sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime') # replace timestamps
|
|
47
|
+
sql.gsub!(/'[^']*'/, ':string') # replace strings
|
|
48
|
+
sql.gsub!(/\b\d+\b/, ':int') # replace integers
|
|
49
|
+
sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list
|
|
50
|
+
sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list
|
|
51
|
+
|
|
52
|
+
return sql.lstrip.rstrip
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def host
|
|
56
|
+
self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Convert the timestamp to an integer
|
|
60
|
+
def convert_timestamp(value, definition)
|
|
61
|
+
all,y,m,d,h,i,s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
|
|
62
|
+
('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -119,13 +119,13 @@ module RequestLogAnalyzer::FileFormat
|
|
|
119
119
|
{ :name => :file, :type => :string }]),
|
|
120
120
|
|
|
121
121
|
:cache_hit => RequestLogAnalyzer::LineDefinition.new(:cache_hit,
|
|
122
|
-
:regexp => /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter/),
|
|
122
|
+
:regexp => /Filter chain halted as \[\#<ActionController::Filters::AroundFilter.*\@method=.*(?:Caching::Actions::ActionCacheFilter|action_controller\/caching\/actions\.rb).*\] did_not_yield/),
|
|
123
123
|
|
|
124
124
|
:parameters => RequestLogAnalyzer::LineDefinition.new(:parameters,
|
|
125
125
|
:teaser => / Parameters:/,
|
|
126
126
|
:regexp => / Parameters:\s+(\{.*\})/,
|
|
127
127
|
:captures => [{ :name => :params, :type => :eval }]),
|
|
128
|
-
|
|
128
|
+
|
|
129
129
|
:rendered => RequestLogAnalyzer::LineDefinition.new(:rendered,
|
|
130
130
|
:teaser => /Rendered /,
|
|
131
131
|
:regexp => /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
|
|
@@ -38,7 +38,7 @@ module RequestLogAnalyzer::Output
|
|
|
38
38
|
foreground_color = "3#{COLORS[option]}" if COLORS.include?(option)
|
|
39
39
|
font_style = "#{STYLES[option]};" if STYLES.include?(option)
|
|
40
40
|
elsif option.kind_of?(Hash)
|
|
41
|
-
|
|
41
|
+
option.each do |key, value|
|
|
42
42
|
case key
|
|
43
43
|
when :color; foreground_color = "3#{COLORS[value]}" if COLORS.include?(value)
|
|
44
44
|
when :background; background_color = "4#{COLORS[value]};" if COLORS.include?(value)
|
data/lib/request_log_analyzer.rb
CHANGED
|
@@ -11,7 +11,7 @@ module RequestLogAnalyzer
|
|
|
11
11
|
|
|
12
12
|
# The current version of request-log-analyzer.
|
|
13
13
|
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
|
14
|
-
VERSION = "1.
|
|
14
|
+
VERSION = "1.6.0"
|
|
15
15
|
|
|
16
16
|
# Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
|
|
17
17
|
# <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
|
|
@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
|
|
|
2
2
|
s.name = "request-log-analyzer"
|
|
3
3
|
|
|
4
4
|
# Do not set the version and date field manually, this is done by the release script
|
|
5
|
-
s.version = "1.
|
|
6
|
-
s.date = "
|
|
5
|
+
s.version = "1.6.0"
|
|
6
|
+
s.date = "2010-01-08"
|
|
7
7
|
|
|
8
8
|
s.rubyforge_project = 'r-l-a'
|
|
9
9
|
|
|
@@ -36,6 +36,6 @@ Gem::Specification.new do |s|
|
|
|
36
36
|
|
|
37
37
|
# The files and test_files directives are set automatically by the release script.
|
|
38
38
|
# Do not change them by hand, but make sure to add the files to the git repository.
|
|
39
|
-
s.files = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/
|
|
40
|
-
s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/
|
|
39
|
+
s.files = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/postgresql.log spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/fixtures/test_language_combined.log spec/unit/request_spec.rb lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/output/fancy_html.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log spec/unit/mailer_spec.rb lib/request_log_analyzer/file_format/rails.rb Rakefile lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/fixtures/sinatra.log spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
|
|
40
|
+
s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
|
|
41
41
|
end
|