request-log-analyzer 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +2 -1
- data/.infinity_test +8 -0
- data/bin/request-log-analyzer +0 -1
- data/lib/request_log_analyzer/aggregator/database_inserter.rb +2 -1
- data/lib/request_log_analyzer/aggregator.rb +5 -5
- data/lib/request_log_analyzer/controller.rb +0 -3
- data/lib/request_log_analyzer/database.rb +6 -7
- data/lib/request_log_analyzer/file_format/apache.rb +1 -1
- data/lib/request_log_analyzer/file_format/delayed_job2.rb +2 -2
- data/lib/request_log_analyzer/file_format/delayed_job21.rb +2 -2
- data/lib/request_log_analyzer/file_format/haproxy.rb +107 -13
- data/lib/request_log_analyzer/file_format/mysql.rb +5 -5
- data/lib/request_log_analyzer/file_format/rails3.rb +8 -1
- data/lib/request_log_analyzer/file_format.rb +43 -14
- data/lib/request_log_analyzer/filter.rb +4 -5
- data/lib/request_log_analyzer/line_definition.rb +6 -4
- data/lib/request_log_analyzer/output.rb +3 -5
- data/lib/request_log_analyzer/source/log_parser.rb +56 -4
- data/lib/request_log_analyzer/source.rb +3 -4
- data/lib/request_log_analyzer/tracker.rb +8 -8
- data/lib/request_log_analyzer.rb +15 -29
- data/request-log-analyzer.gemspec +3 -3
- data/spec/fixtures/mysql_slow_query.log +0 -1
- data/spec/integration/command_line_usage_spec.rb +0 -5
- data/spec/lib/helpers.rb +2 -2
- data/spec/lib/matchers.rb +38 -7
- data/spec/lib/mocks.rb +1 -5
- data/spec/unit/database/base_class_spec.rb +1 -0
- data/spec/unit/file_format/amazon_s3_format_spec.rb +58 -55
- data/spec/unit/file_format/apache_format_spec.rb +74 -162
- data/spec/unit/file_format/common_regular_expressions_spec.rb +51 -26
- data/spec/unit/file_format/delayed_job21_format_spec.rb +22 -31
- data/spec/unit/file_format/delayed_job2_format_spec.rb +27 -32
- data/spec/unit/file_format/delayed_job_format_spec.rb +44 -63
- data/spec/unit/file_format/haproxy_format_spec.rb +69 -71
- data/spec/unit/file_format/line_definition_spec.rb +26 -33
- data/spec/unit/file_format/merb_format_spec.rb +22 -37
- data/spec/unit/file_format/mysql_format_spec.rb +80 -123
- data/spec/unit/file_format/oink_format_spec.rb +29 -61
- data/spec/unit/file_format/postgresql_format_spec.rb +2 -4
- data/spec/unit/file_format/rack_format_spec.rb +49 -44
- data/spec/unit/file_format/rails3_format_spec.rb +39 -37
- data/spec/unit/file_format/rails_format_spec.rb +52 -68
- data/spec/unit/file_format/w3c_format_spec.rb +40 -39
- data/spec/unit/source/log_parser_spec.rb +1 -1
- metadata +4 -6
- data/lib/mixins/gets_memory_protection.rb +0 -80
- data/lib/request_log_analyzer/output/fancy_html.rb +0 -44
- data/lib/request_log_analyzer/source/database_loader.rb +0 -87
|
@@ -1,12 +1,5 @@
|
|
|
1
1
|
module RequestLogAnalyzer::Tracker
|
|
2
2
|
|
|
3
|
-
autoload :Duration, 'request_log_analyzer/tracker/duration'
|
|
4
|
-
autoload :Frequency, 'request_log_analyzer/tracker/frequency'
|
|
5
|
-
autoload :HourlySpread, 'request_log_analyzer/tracker/hourly_spread'
|
|
6
|
-
autoload :NumericValue, 'request_log_analyzer/tracker/numeric_value'
|
|
7
|
-
autoload :Timespan, 'request_log_analyzer/tracker/timespan'
|
|
8
|
-
autoload :Traffic, 'request_log_analyzer/tracker/traffic'
|
|
9
|
-
|
|
10
3
|
# Base Tracker class. All other trackers inherit from this class
|
|
11
4
|
#
|
|
12
5
|
# Accepts the following options:
|
|
@@ -99,4 +92,11 @@ module RequestLogAnalyzer::Tracker
|
|
|
99
92
|
nil
|
|
100
93
|
end
|
|
101
94
|
end
|
|
102
|
-
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
require 'request_log_analyzer/tracker/numeric_value'
|
|
98
|
+
require 'request_log_analyzer/tracker/duration'
|
|
99
|
+
require 'request_log_analyzer/tracker/frequency'
|
|
100
|
+
require 'request_log_analyzer/tracker/hourly_spread'
|
|
101
|
+
require 'request_log_analyzer/tracker/timespan'
|
|
102
|
+
require 'request_log_analyzer/tracker/traffic'
|
data/lib/request_log_analyzer.rb
CHANGED
|
@@ -10,36 +10,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
|
|
|
10
10
|
# The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
|
|
11
11
|
# is running.
|
|
12
12
|
module RequestLogAnalyzer
|
|
13
|
-
|
|
13
|
+
|
|
14
14
|
# The current version of request-log-analyzer.
|
|
15
15
|
# Do not change the value by hand; it will be updated automatically by the gem release script.
|
|
16
|
-
VERSION = "1.
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
autoload :Controller, 'request_log_analyzer/controller'
|
|
20
|
-
autoload :Aggregator, 'request_log_analyzer/aggregator'
|
|
21
|
-
autoload :Database, 'request_log_analyzer/database'
|
|
22
|
-
autoload :FileFormat, 'request_log_analyzer/file_format'
|
|
23
|
-
autoload :Filter, 'request_log_analyzer/filter'
|
|
24
|
-
autoload :LineDefinition, 'request_log_analyzer/line_definition'
|
|
25
|
-
autoload :LogProcessor, 'request_log_analyzer/log_processor'
|
|
26
|
-
autoload :Mailer, 'request_log_analyzer/mailer'
|
|
27
|
-
autoload :Output, 'request_log_analyzer/output'
|
|
28
|
-
autoload :Request, 'request_log_analyzer/request'
|
|
29
|
-
autoload :Source, 'request_log_analyzer/source'
|
|
30
|
-
autoload :Tracker, 'request_log_analyzer/tracker'
|
|
31
|
-
|
|
32
|
-
# Loads constants that reside in the RequestLogAnalyzer tree using the constant name and its base
|
|
33
|
-
# constant to determine the filename.
|
|
34
|
-
# @param [Module] base The base constant to load the constant from. This should be <tt>Foo</tt> when
|
|
35
|
-
# the constant <tt>Foo::Bar</tt> is being loaded.
|
|
36
|
-
# @param [Symbol] const The constant to load from the base constant as a string or symbol. This
|
|
37
|
-
# should be <tt>"Bar"<tt> or <tt>:Bar</tt> when the constant <tt>Foo::Bar</tt> is being loaded.
|
|
38
|
-
# @return [Module] The loaded module, nil if it was not found on the expected location.
|
|
39
|
-
def self.load_default_class_file(base, const)
|
|
40
|
-
require "#{to_underscore("#{base.name}::#{const}")}"
|
|
41
|
-
base.const_get(const) if base.const_defined?(const)
|
|
42
|
-
end
|
|
16
|
+
VERSION = "1.11.0"
|
|
43
17
|
|
|
44
18
|
# Convert a string/symbol in camelcase ({RequestLogAnalyzer::Controller}) to underscores
|
|
45
19
|
# (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using
|
|
@@ -55,9 +29,21 @@ module RequestLogAnalyzer
|
|
|
55
29
|
# ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given
|
|
56
30
|
# filename.
|
|
57
31
|
#
|
|
58
|
-
# @param [#to_s] str The string-like to convert in the
|
|
32
|
+
# @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
|
|
59
33
|
# @return [String] The input string converted to camelcase form.
|
|
60
34
|
def self.to_camelcase(str)
|
|
61
35
|
str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
|
|
62
36
|
end
|
|
63
37
|
end
|
|
38
|
+
|
|
39
|
+
require 'request_log_analyzer/controller'
|
|
40
|
+
require 'request_log_analyzer/aggregator'
|
|
41
|
+
require 'request_log_analyzer/file_format'
|
|
42
|
+
require 'request_log_analyzer/filter'
|
|
43
|
+
require 'request_log_analyzer/line_definition'
|
|
44
|
+
require 'request_log_analyzer/log_processor'
|
|
45
|
+
require 'request_log_analyzer/mailer'
|
|
46
|
+
require 'request_log_analyzer/output'
|
|
47
|
+
require 'request_log_analyzer/request'
|
|
48
|
+
require 'request_log_analyzer/source'
|
|
49
|
+
require 'request_log_analyzer/tracker'
|
|
@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
|
|
|
2
2
|
s.name = "request-log-analyzer"
|
|
3
3
|
|
|
4
4
|
# Do not set the version and date field manually, this is done by the release script
|
|
5
|
-
s.version = "1.
|
|
6
|
-
s.date = "2011-
|
|
5
|
+
s.version = "1.11.0"
|
|
6
|
+
s.date = "2011-04-27"
|
|
7
7
|
|
|
8
8
|
s.rubyforge_project = 'r-l-a'
|
|
9
9
|
|
|
@@ -39,6 +39,6 @@ Gem::Specification.new do |s|
|
|
|
39
39
|
|
|
40
40
|
# The files and test_files directives are set automatically by the release script.
|
|
41
41
|
# Do not change them by hand, but make sure to add the files to the git repository.
|
|
42
|
-
s.files = %w(.gitignore DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/
|
|
42
|
+
s.files = %w(.gitignore .infinity_test DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
|
|
43
43
|
s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
|
|
44
44
|
end
|
|
@@ -14,11 +14,6 @@ describe RequestLogAnalyzer, 'running from command line' do
|
|
|
14
14
|
output = run("#{log_fixture(:rails_1x)}")
|
|
15
15
|
output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
|
|
16
16
|
end
|
|
17
|
-
|
|
18
|
-
it "should function correctly with the gets mixin" do
|
|
19
|
-
output = run("#{log_fixture(:rails_1x)} --gets-memory-protection")
|
|
20
|
-
output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
|
|
21
|
-
end
|
|
22
17
|
|
|
23
18
|
it "should find 2 requests when parsing a compressed file" do
|
|
24
19
|
output = run("#{log_fixture(:decompression, :tgz)}")
|
data/spec/lib/helpers.rb
CHANGED
|
@@ -11,8 +11,8 @@ module RequestLogAnalyzer::RSpec::Helpers
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
# Creates a log file given some lines
|
|
14
|
-
def
|
|
15
|
-
StringIO.new(lines.join("\n")
|
|
14
|
+
def log_snippet(*lines)
|
|
15
|
+
StringIO.new(lines.join("\n") << "\n")
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
# Request loopback
|
data/spec/lib/matchers.rb
CHANGED
|
@@ -7,15 +7,23 @@ module RequestLogAnalyzer::RSpec::Matchers
|
|
|
7
7
|
@captures = []
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
def
|
|
10
|
+
def and_capture(*captures)
|
|
11
11
|
@captures += captures
|
|
12
12
|
return self
|
|
13
13
|
end
|
|
14
|
+
|
|
15
|
+
alias_method :capturing, :and_capture
|
|
16
|
+
|
|
17
|
+
def description
|
|
18
|
+
description = "have a #{@line_type.inspect} line definition"
|
|
19
|
+
description << " that captures #{@captures.join(', ')}" unless @captures.empty?
|
|
20
|
+
description
|
|
21
|
+
end
|
|
14
22
|
|
|
15
23
|
def matches?(file_format)
|
|
16
24
|
file_format = file_format.create if file_format.kind_of?(Class)
|
|
17
25
|
if ld = file_format.line_definitions[@line_type]
|
|
18
|
-
@captures.all? { |c| ld.
|
|
26
|
+
@captures.all? { |c| ld.all_captured_variables.include?(c) }
|
|
19
27
|
else
|
|
20
28
|
false
|
|
21
29
|
end
|
|
@@ -24,10 +32,25 @@ module RequestLogAnalyzer::RSpec::Matchers
|
|
|
24
32
|
|
|
25
33
|
class ParseLine
|
|
26
34
|
|
|
27
|
-
def initialize(line)
|
|
35
|
+
def initialize(line, line_description = nil)
|
|
28
36
|
@line = line
|
|
29
37
|
@captures = {}
|
|
30
38
|
@line_type = nil
|
|
39
|
+
@line_description = line_description
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def line_description
|
|
43
|
+
@full_line_description ||= if @line_description
|
|
44
|
+
if @line_type && @line_description =~ /^(?:with|without|having|using) /
|
|
45
|
+
"a #{@line_type.inspect} line #{@line_description}"
|
|
46
|
+
else
|
|
47
|
+
@line_description
|
|
48
|
+
end
|
|
49
|
+
elsif @line_type
|
|
50
|
+
"a #{@line_type.inspect} line"
|
|
51
|
+
else
|
|
52
|
+
"line #{@line.inspect}"
|
|
53
|
+
end
|
|
31
54
|
end
|
|
32
55
|
|
|
33
56
|
def failure_message
|
|
@@ -43,11 +66,20 @@ module RequestLogAnalyzer::RSpec::Matchers
|
|
|
43
66
|
@captures = captures
|
|
44
67
|
return self
|
|
45
68
|
end
|
|
69
|
+
|
|
70
|
+
alias_method :capturing, :and_capture
|
|
46
71
|
|
|
47
72
|
def fail(message)
|
|
48
73
|
@failure_message = message
|
|
49
74
|
return false
|
|
50
75
|
end
|
|
76
|
+
|
|
77
|
+
def description
|
|
78
|
+
description = "parse #{line_description}"
|
|
79
|
+
description << " as line type #{@line_type.inspect}" if @line_type
|
|
80
|
+
description << " and capture #{@captures.keys.join(', ')} correctly" unless @captures.empty?
|
|
81
|
+
description
|
|
82
|
+
end
|
|
51
83
|
|
|
52
84
|
def matches?(file_format)
|
|
53
85
|
if @line_hash = file_format.parse_line(@line)
|
|
@@ -70,8 +102,7 @@ module RequestLogAnalyzer::RSpec::Matchers
|
|
|
70
102
|
return HasLineDefinition.new(line_type)
|
|
71
103
|
end
|
|
72
104
|
|
|
73
|
-
def parse_line(line)
|
|
74
|
-
ParseLine.new(line)
|
|
105
|
+
def parse_line(line, line_description = nil)
|
|
106
|
+
ParseLine.new(line, line_description)
|
|
75
107
|
end
|
|
76
|
-
|
|
77
|
-
end
|
|
108
|
+
end
|
data/spec/lib/mocks.rb
CHANGED
|
@@ -1,67 +1,70 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
4
|
+
|
|
5
|
+
subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
|
|
4
6
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@sample = '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"'
|
|
9
|
-
end
|
|
7
|
+
it { should be_well_formed }
|
|
8
|
+
it { should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
|
|
9
|
+
:key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent) }
|
|
10
10
|
|
|
11
|
-
it
|
|
12
|
-
@file_format.should be_valid
|
|
13
|
-
end
|
|
11
|
+
it { should have(7).report_trackers }
|
|
14
12
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
let(:sample_get) { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
|
|
14
|
+
let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
|
|
15
|
+
|
|
16
|
+
describe '#parse_line' do
|
|
17
|
+
it { should parse_line(sample_get, 'a GET line').and_capture(
|
|
18
|
+
:bucket_owner => '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
|
|
19
|
+
:bucket => 'sample-bucket',
|
|
20
|
+
:timestamp => 20061006014214,
|
|
21
|
+
:remote_ip => '207.171.172.6',
|
|
22
|
+
:key => 'object.png',
|
|
23
|
+
:operation => 'REST.GET.OBJECT',
|
|
24
|
+
:requester => '65a011a29cdf8ec533ec3d1ccaae921c',
|
|
25
|
+
:request_id => 'C980091AD89C936A',
|
|
26
|
+
:request_uri => 'GET /sample-bucket/object.png HTTP/1.1',
|
|
27
|
+
:error_code => nil,
|
|
28
|
+
:http_status => 200,
|
|
29
|
+
:total_time => 0.988,
|
|
30
|
+
:turnaround_time => 0.987,
|
|
31
|
+
:bytes_sent => 1243,
|
|
32
|
+
:object_size => 1243,
|
|
33
|
+
:user_agent => 'aranhabot',
|
|
34
|
+
:referer => nil)
|
|
35
|
+
}
|
|
19
36
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
37
|
+
it { should parse_line(sample_copy, 'a COPY line').and_capture(
|
|
38
|
+
:bucket_owner => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
39
|
+
:bucket => 'testbucket',
|
|
40
|
+
:timestamp => 20100303230459,
|
|
41
|
+
:remote_ip => '174.119.31.76',
|
|
42
|
+
:key => 'files/image.png',
|
|
43
|
+
:operation => 'REST.COPY.OBJECT',
|
|
44
|
+
:requester => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
45
|
+
:request_id => 'ACCC34B843C87BC9',
|
|
46
|
+
:request_uri => 'PUT /files/image.png HTTP/1.1',
|
|
47
|
+
:error_code => nil,
|
|
48
|
+
:http_status => 200,
|
|
49
|
+
:total_time => 0.365,
|
|
50
|
+
:turnaround_time => 0.319,
|
|
51
|
+
:bytes_sent => 234,
|
|
52
|
+
:object_size => 65957,
|
|
53
|
+
:user_agent => '',
|
|
54
|
+
:referer => nil)
|
|
55
|
+
}
|
|
23
56
|
|
|
24
|
-
|
|
25
|
-
@file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
|
|
57
|
+
it { should_not parse_line('nonsense', 'a nonsense line') }
|
|
26
58
|
end
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
request[:request_id].should == 'C980091AD89C936A'
|
|
37
|
-
request[:request_uri].should == 'GET /sample-bucket/object.png HTTP/1.1'
|
|
38
|
-
request[:error_code].should == nil
|
|
39
|
-
request[:http_status].should == 200
|
|
40
|
-
request[:total_time].should == 0.988
|
|
41
|
-
request[:turnaround_time].should == 0.987
|
|
42
|
-
request[:bytes_sent].should == 1243
|
|
43
|
-
request[:object_size].should == 1243
|
|
44
|
-
request[:user_agent].should == 'aranhabot'
|
|
45
|
-
request[:referer].should == nil
|
|
59
|
+
|
|
60
|
+
describe '#parse_io' do
|
|
61
|
+
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
|
62
|
+
let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
|
|
63
|
+
|
|
64
|
+
it "should parse requests correctly and not generate warnings" do
|
|
65
|
+
log_parser.should_receive(:handle_request).twice
|
|
66
|
+
log_parser.should_not_receive(:warn)
|
|
67
|
+
log_parser.parse_io(snippet)
|
|
46
68
|
end
|
|
47
69
|
end
|
|
48
|
-
|
|
49
|
-
it "should parse a COPY request correctly" do
|
|
50
|
-
line = '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -'
|
|
51
|
-
@file_format.should parse_line(line).as(:access).and_capture(
|
|
52
|
-
:bucket_owner => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
53
|
-
:bucket => 'testbucket',
|
|
54
|
-
:timestamp => 20100303230459,
|
|
55
|
-
:remote_ip => '174.119.31.76',
|
|
56
|
-
:requester => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
|
57
|
-
:key => 'files/image.png',
|
|
58
|
-
:operation => 'REST.COPY.OBJECT',
|
|
59
|
-
:total_time => 0.365,
|
|
60
|
-
:turnaround_time => 0.319,
|
|
61
|
-
:bytes_sent => 234,
|
|
62
|
-
:object_size => 65957,
|
|
63
|
-
:referer => nil,
|
|
64
|
-
:user_agent => '')
|
|
65
|
-
end
|
|
66
|
-
|
|
67
70
|
end
|