request-log-analyzer 1.10.1 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. data/bin/request-log-analyzer +0 -1
  2. data/lib/request_log_analyzer.rb +15 -29
  3. data/lib/request_log_analyzer/aggregator.rb +5 -5
  4. data/lib/request_log_analyzer/aggregator/database_inserter.rb +2 -1
  5. data/lib/request_log_analyzer/controller.rb +0 -3
  6. data/lib/request_log_analyzer/database.rb +6 -7
  7. data/lib/request_log_analyzer/file_format.rb +42 -13
  8. data/lib/request_log_analyzer/file_format/apache.rb +1 -1
  9. data/lib/request_log_analyzer/file_format/delayed_job2.rb +2 -2
  10. data/lib/request_log_analyzer/file_format/delayed_job21.rb +2 -2
  11. data/lib/request_log_analyzer/file_format/haproxy.rb +107 -13
  12. data/lib/request_log_analyzer/file_format/mysql.rb +5 -5
  13. data/lib/request_log_analyzer/file_format/rails3.rb +7 -0
  14. data/lib/request_log_analyzer/filter.rb +4 -5
  15. data/lib/request_log_analyzer/line_definition.rb +6 -4
  16. data/lib/request_log_analyzer/output.rb +3 -5
  17. data/lib/request_log_analyzer/source.rb +3 -4
  18. data/lib/request_log_analyzer/source/log_parser.rb +56 -4
  19. data/lib/request_log_analyzer/tracker.rb +8 -8
  20. data/request-log-analyzer.gemspec +3 -3
  21. data/spec/fixtures/mysql_slow_query.log +0 -1
  22. data/spec/integration/command_line_usage_spec.rb +0 -5
  23. data/spec/lib/helpers.rb +2 -2
  24. data/spec/lib/matchers.rb +38 -7
  25. data/spec/lib/mocks.rb +1 -5
  26. data/spec/unit/database/base_class_spec.rb +1 -0
  27. data/spec/unit/file_format/amazon_s3_format_spec.rb +58 -55
  28. data/spec/unit/file_format/apache_format_spec.rb +74 -162
  29. data/spec/unit/file_format/common_regular_expressions_spec.rb +51 -26
  30. data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -31
  31. data/spec/unit/file_format/delayed_job2_format_spec.rb +27 -32
  32. data/spec/unit/file_format/delayed_job_format_spec.rb +44 -63
  33. data/spec/unit/file_format/haproxy_format_spec.rb +69 -71
  34. data/spec/unit/file_format/line_definition_spec.rb +26 -33
  35. data/spec/unit/file_format/merb_format_spec.rb +22 -37
  36. data/spec/unit/file_format/mysql_format_spec.rb +80 -123
  37. data/spec/unit/file_format/oink_format_spec.rb +29 -61
  38. data/spec/unit/file_format/postgresql_format_spec.rb +2 -4
  39. data/spec/unit/file_format/rack_format_spec.rb +49 -44
  40. data/spec/unit/file_format/rails3_format_spec.rb +17 -20
  41. data/spec/unit/file_format/rails_format_spec.rb +52 -68
  42. data/spec/unit/file_format/w3c_format_spec.rb +40 -39
  43. data/spec/unit/source/log_parser_spec.rb +1 -1
  44. metadata +4 -7
  45. data/lib/mixins/gets_memory_protection.rb +0 -80
  46. data/lib/request_log_analyzer/output/fancy_html.rb +0 -44
  47. data/lib/request_log_analyzer/source/database_loader.rb +0 -87
@@ -11,7 +11,7 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
11
11
  end
12
12
 
13
13
  it "should have a valid language" do
14
- @log_parser.file_format.should be_valid
14
+ @log_parser.file_format.should be_well_formed
15
15
  end
16
16
 
17
17
  it "should set the :source for every parsed line" do
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 1
7
- - 10
8
- - 1
9
- version: 1.10.1
7
+ - 11
8
+ - 0
9
+ version: 1.11.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Willem van Bergen
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2011-03-17 00:00:00 -04:00
18
+ date: 2011-04-27 00:00:00 -04:00
19
19
  default_executable: request-log-analyzer
20
20
  dependencies:
21
21
  - !ruby/object:Gem::Dependency
@@ -95,7 +95,6 @@ files:
95
95
  - lib/cli/database_console_init.rb
96
96
  - lib/cli/progressbar.rb
97
97
  - lib/cli/tools.rb
98
- - lib/mixins/gets_memory_protection.rb
99
98
  - lib/request_log_analyzer.rb
100
99
  - lib/request_log_analyzer/aggregator.rb
101
100
  - lib/request_log_analyzer/aggregator/database_inserter.rb
@@ -132,12 +131,10 @@ files:
132
131
  - lib/request_log_analyzer/log_processor.rb
133
132
  - lib/request_log_analyzer/mailer.rb
134
133
  - lib/request_log_analyzer/output.rb
135
- - lib/request_log_analyzer/output/fancy_html.rb
136
134
  - lib/request_log_analyzer/output/fixed_width.rb
137
135
  - lib/request_log_analyzer/output/html.rb
138
136
  - lib/request_log_analyzer/request.rb
139
137
  - lib/request_log_analyzer/source.rb
140
- - lib/request_log_analyzer/source/database_loader.rb
141
138
  - lib/request_log_analyzer/source/log_parser.rb
142
139
  - lib/request_log_analyzer/tracker.rb
143
140
  - lib/request_log_analyzer/tracker/duration.rb
@@ -1,80 +0,0 @@
1
- class File
2
- alias_method :gets_original, :gets
3
- # The size of the reads we will use to add to the line buffer.
4
- MAX_READ_SIZE=1024*100
5
-
6
- #
7
- # This method returns the next line of the File.
8
- #
9
- # It works by moving the file pointer forward +MAX_READ_SIZE+ at a time,
10
- # storing seen lines in <tt>@line_buffer</tt>. Once the buffer contains at
11
- # least two lines (ensuring we have seen on full line) or the file pointer
12
- # reaches the end of the File, the last line from the buffer is returned.
13
- # When the buffer is exhausted, this will throw +nil+ (from the empty Array).
14
- #
15
- # Read portions of the file that do not contain the +sep_string+ are not added to
16
- # the buffer. This prevents <tt>@line_buffer<tt> from growing signficantly when parsing
17
- # large lines.
18
- #
19
- def gets(sep_string = $/)
20
- @read_size ||= MAX_READ_SIZE
21
- # A buffer to hold lines read, but not yet returned.
22
- @line_buffer ||= Array.new
23
-
24
- # Record where we are.
25
- @current_pos ||= pos
26
-
27
- # Last Position in the file
28
- @last_pos ||= nil
29
- if @last_pos.nil?
30
- seek(0, IO::SEEK_END)
31
- @last_pos = pos
32
- seek(@current_pos, IO::SEEK_SET)
33
- end
34
-
35
- #
36
- # If we have more than one line in the buffer or we have reached the
37
- # beginning of the file, send the last line in the buffer to the caller.
38
- # (This may be +nil+, if the buffer has been exhausted.)
39
- #
40
- if @line_buffer.size > 2 or @current_pos >= @last_pos
41
- self.lineno += 1
42
- return @line_buffer.shift
43
- end
44
-
45
- sep =
46
-
47
- chunk = String.new
48
- while chunk and chunk !~ /#{sep_string}/
49
- chunk = read(@read_size)
50
- end
51
-
52
- # Appends new lines to the last element of the buffer
53
- line_buffer_pos = @line_buffer.any? ? @line_buffer.size-1 : 0
54
-
55
- if chunk
56
- @line_buffer[line_buffer_pos] = @line_buffer[line_buffer_pos].to_s<< chunk
57
- else
58
- # at the end
59
- return @line_buffer.shift
60
- end
61
-
62
- #
63
- # Divide the last line of the buffer based on +sep_string+ and #flatten!
64
- # those new lines into the buffer.
65
- #
66
- @line_buffer[line_buffer_pos] = @line_buffer[line_buffer_pos].scan(/.*?#{Regexp.escape(sep_string)}|.+/)
67
- @line_buffer.flatten!
68
-
69
- #
70
- # If we made it this far, we need to read more data to try and find the
71
- # end of a line or the end of the file. Move the file pointer
72
- # forward a step, to give us new bytes to read.
73
- #
74
- @current_pos += @read_size
75
- seek(@current_pos, IO::SEEK_SET)
76
-
77
- # We have more data now, so try again to read a line...
78
- gets(sep_string)
79
- end
80
- end
@@ -1,44 +0,0 @@
1
- begin
2
- require 'rubygems'
3
- require 'gchart'
4
- rescue LoadError
5
- $stderr.puts "The FancyHTML output format requires the googlechart gem:"
6
- $stderr.puts " (sudo) gem install googlecharts"
7
- end
8
-
9
- module RequestLogAnalyzer::Output
10
-
11
- class FancyHTML < HTML
12
-
13
- def report_tracker(tracker)
14
- case tracker
15
- when RequestLogAnalyzer::Tracker::HourlySpread then report_hourly_spread(tracker)
16
- else tracker.report(self)
17
- end
18
- end
19
-
20
- def report_hourly_spread(tracker)
21
- title tracker.title
22
- puts tag(:img, nil, :width => '700', :height => '120', :src =>
23
- Gchart.sparkline(:data => tracker.hour_frequencies, :size => '700x120', :line_colors => '0077CC',
24
- :axis_with_labels => 'x,y', :axis_labels => [x_axis_labels(tracker),y_axis_labels(tracker)]))
25
- end
26
-
27
- def x_axis_labels(tracker)
28
- frmt = '%H:%M'
29
- x_axis_labels = []
30
- num_labels = 5
31
- start = tracker.first_timestamp
32
- step = tracker.timespan / (num_labels - 1)
33
- for i in 0...num_labels do
34
- x_axis_labels << (start + step * i).strftime(frmt)
35
- end
36
- x_axis_labels
37
- end
38
-
39
- def y_axis_labels(tracker)
40
- sorted_frequencies = tracker.hour_frequencies.sort
41
- [sorted_frequencies.first, sorted_frequencies.last]
42
- end
43
- end
44
- end
@@ -1,87 +0,0 @@
1
- require 'rubygems'
2
- require 'activerecord'
3
-
4
- module RequestLogAnalyzer::Source
5
-
6
- # Active Resource hook
7
- class Request < ActiveRecord::Base
8
- has_many :completed_lines
9
- has_many :processing_lines
10
- def convert(file_format)
11
- send_attributes = self.attributes
12
- send_attributes.merge!(self.completed_lines.first.attributes) if self.completed_lines.first
13
- send_attributes.merge!(self.processing_lines.first.attributes) if self.processing_lines.first
14
- return RequestLogAnalyzer::Request.new(file_format, send_attributes)
15
- end
16
- end
17
-
18
- class CompletedLine < ActiveRecord::Base
19
- belongs_to :request
20
- end
21
-
22
- class ProcessingLine < ActiveRecord::Base
23
- belongs_to :request
24
- end
25
-
26
- # The Database class gets log data from the database.
27
- class DatabaseLoader < Base
28
-
29
- attr_reader :source_files, :file_format, :requests
30
-
31
- # Initializes the log file parser instance.
32
- # It will apply the language specific FileFormat module to this instance. It will use the line
33
- # definitions in this module to parse any input that it is given (see parse_io).
34
- #
35
- # <tt>format</tt>:: The current file format instance
36
- # <tt>options</tt>:: A hash of options that are used by the parser
37
- def initialize(format, options = {})
38
- super(format, options)
39
- @source_files = options[:source_files]
40
- @parsed_requests = 0
41
- @requests = []
42
- end
43
-
44
- # Reads the input, which can either be a file, sequence of files or STDIN to parse
45
- # lines specified in the FileFormat. This lines will be combined into Request instances,
46
- # that will be yielded. The actual parsing occurs in the parse_io method.
47
- # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
48
- def each_request(options = {}, &block) # :yields: request
49
- ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => @source_files)
50
-
51
- @progress_handler.call(:started, @source_files) if @progress_handler
52
- RequestLogAnalyzer::Source::Request.find(:all).each do |request|
53
- @parsed_requests += 1
54
- @progress_handler.call(:progress, @parsed_requests) if @progress_handler
55
-
56
- yield request.convert(self.file_format)
57
- end
58
-
59
- @progress_handler.call(:finished, @source_files) if @progress_handler
60
- end
61
-
62
- # Add a block to this method to install a progress handler while parsing.
63
- # <tt>proc</tt>:: The proc that will be called to handle progress update messages
64
- def progress=(proc)
65
- @progress_handler = proc
66
- end
67
-
68
- # Add a block to this method to install a warning handler while parsing,
69
- # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
70
- def warning=(proc)
71
- @warning_handler = proc
72
- end
73
-
74
- # This method is called by the parser if it encounteres any parsing problems.
75
- # It will call the installed warning handler if any.
76
- #
77
- # By default, RequestLogAnalyzer::Controller will install a warning handler
78
- # that will pass the warnings to each aggregator so they can do something useful
79
- # with it.
80
- #
81
- # <tt>type</tt>:: The warning type (a Symbol)
82
- # <tt>message</tt>:: A message explaining the warning
83
- def warn(type, message)
84
- @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
85
- end
86
- end
87
- end