request-log-analyzer 1.12.9 → 1.12.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +13 -0
- data/Gemfile.activerecord2 +5 -0
- data/Gemfile.activerecord3 +4 -0
- data/Gemfile.activerecord4 +4 -0
- data/LICENSE +1 -1
- data/README.rdoc +1 -2
- data/lib/request_log_analyzer/file_format/amazon_s3.rb +3 -3
- data/lib/request_log_analyzer/file_format/apache.rb +6 -5
- data/lib/request_log_analyzer/version.rb +1 -1
- data/request-log-analyzer.gemspec +2 -1
- data/spec/lib/helpers.rb +1 -1
- data/spec/lib/mocks.rb +43 -43
- data/spec/unit/aggregator/database_inserter_spec.rb +1 -1
- data/spec/unit/controller/controller_spec.rb +4 -4
- data/spec/unit/database/base_class_spec.rb +19 -19
- data/spec/unit/database/database_spec.rb +7 -7
- data/spec/unit/file_format/amazon_s3_format_spec.rb +62 -40
- data/spec/unit/file_format/apache_format_spec.rb +34 -20
- data/spec/unit/file_format/line_definition_spec.rb +3 -3
- metadata +11 -7
data/.travis.yml
CHANGED
@@ -14,10 +14,23 @@ rvm:
|
|
14
14
|
- jruby-head
|
15
15
|
- rbx-18mode
|
16
16
|
- rbx-19mode
|
17
|
+
gemfile:
|
18
|
+
- Gemfile.activerecord2
|
19
|
+
- Gemfile.activerecord3
|
20
|
+
- Gemfile.activerecord4
|
17
21
|
matrix:
|
18
22
|
allow_failures:
|
19
23
|
- rvm: jruby-head
|
20
24
|
- rvm: ruby-head
|
25
|
+
exclude:
|
26
|
+
- gemfile: Gemfile.activerecord4
|
27
|
+
rvm: 1.8.7
|
28
|
+
- gemfile: Gemfile.activerecord4
|
29
|
+
rvm: ree
|
30
|
+
- gemfile: Gemfile.activerecord4
|
31
|
+
rvm: jruby-18mode
|
32
|
+
- gemfile: Gemfile.activerecord4
|
33
|
+
rvm: rbx-18mode
|
21
34
|
notifications:
|
22
35
|
email:
|
23
36
|
- info@railsdoctors.com
|
data/LICENSE
CHANGED
data/README.rdoc
CHANGED
@@ -47,5 +47,4 @@ Bergen (willem@railsdoctors.com) or Bart ten Brinke (bart@railsdoctors.com).
|
|
47
47
|
|
48
48
|
* Project wiki at GitHub: http://github.com/wvanbergen/request-log-analyzer/wiki
|
49
49
|
* Issue tracker at GitHub: http://github.com/wvanbergen/request-log-analyzer/issues
|
50
|
-
* railsdoctors homepage: http://railsdoctors.com
|
51
|
-
* wvanbergen's blog posts: http://techblog.floorplanner.com/tag/request-log-analyzer
|
50
|
+
* railsdoctors homepage: http://railsdoctors.com
|
@@ -5,14 +5,14 @@ module RequestLogAnalyzer::FileFormat
|
|
5
5
|
# Access logs are disabled by default on Amazon S3. To enable logging, see
|
6
6
|
# http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
|
7
7
|
class AmazonS3 < Base
|
8
|
-
|
8
|
+
|
9
9
|
extend CommonRegularExpressions
|
10
10
|
|
11
11
|
line_definition :access do |line|
|
12
12
|
line.header = true
|
13
13
|
line.footer = true
|
14
|
-
line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (
|
15
|
-
|
14
|
+
line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) ([^\ ]+) (\d+) (\d+) ([^\ ]+) "([^"]*)" "([^"]*)"/
|
15
|
+
|
16
16
|
line.capture(:bucket_owner)
|
17
17
|
line.capture(:bucket)
|
18
18
|
line.capture(:timestamp).as(:timestamp)
|
@@ -20,6 +20,7 @@ module RequestLogAnalyzer::FileFormat
|
|
20
20
|
LOG_FORMAT_DEFAULTS = {
|
21
21
|
:common => '%h %l %u %t "%r" %>s %b',
|
22
22
|
:combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
|
23
|
+
:vhost_combined => '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
|
23
24
|
:nginx => '%a %t %h %u "%r" %>s %b',
|
24
25
|
:rack => '%h %l %u %t "%r" %>s %b %T',
|
25
26
|
:referer => '%{Referer}i -> %U',
|
@@ -32,6 +33,7 @@ module RequestLogAnalyzer::FileFormat
|
|
32
33
|
# A hash that defines how the log format directives should be parsed.
|
33
34
|
LOG_DIRECTIVES = {
|
34
35
|
'%' => { nil => { :regexp => '%', :captures => [] } },
|
36
|
+
'v' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :vhost, :type => :string}] } },
|
35
37
|
'h' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :remote_host, :type => :string}] } },
|
36
38
|
'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
|
37
39
|
'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
|
@@ -41,7 +43,7 @@ module RequestLogAnalyzer::FileFormat
|
|
41
43
|
'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
|
42
44
|
},
|
43
45
|
'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
|
44
|
-
'T' => { nil => { :regexp => '(
|
46
|
+
'T' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
|
45
47
|
't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
|
46
48
|
's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
|
47
49
|
'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
|
@@ -66,12 +68,11 @@ module RequestLogAnalyzer::FileFormat
|
|
66
68
|
def self.access_line_definition(format_string)
|
67
69
|
format_string ||= :common
|
68
70
|
format_string = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
|
69
|
-
|
70
|
-
|
71
|
+
|
71
72
|
line_regexp = ''
|
72
73
|
captures = []
|
73
74
|
format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
|
74
|
-
|
75
|
+
|
75
76
|
line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
|
76
77
|
|
77
78
|
if variable
|
@@ -87,7 +88,7 @@ module RequestLogAnalyzer::FileFormat
|
|
87
88
|
end
|
88
89
|
end
|
89
90
|
end
|
90
|
-
|
91
|
+
|
91
92
|
# Return a new line definition object
|
92
93
|
return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
|
93
94
|
:captures => captures, :header => true, :footer => true)
|
@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.authors = ['Willem van Bergen', 'Bart ten Brinke']
|
13
13
|
gem.email = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
|
14
14
|
gem.homepage = 'http://railsdoctors.com'
|
15
|
+
gem.license = "MIT"
|
15
16
|
|
16
17
|
gem.summary = "A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers"
|
17
18
|
gem.description = <<-eos
|
@@ -29,7 +30,7 @@ Gem::Specification.new do |gem|
|
|
29
30
|
|
30
31
|
gem.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required."
|
31
32
|
gem.add_development_dependency('rake')
|
32
|
-
gem.add_development_dependency('rspec', '~> 2.
|
33
|
+
gem.add_development_dependency('rspec', '~> 2.14')
|
33
34
|
gem.add_development_dependency('activerecord')
|
34
35
|
if defined?(JRUBY_VERSION)
|
35
36
|
gem.add_development_dependency('jdbc-sqlite3')
|
data/spec/lib/helpers.rb
CHANGED
@@ -36,7 +36,7 @@ module RequestLogAnalyzer::RSpec::Helpers
|
|
36
36
|
arguments = arguments.join(' ') if arguments.kind_of?(Array)
|
37
37
|
|
38
38
|
output = []
|
39
|
-
IO.popen("#{binary} #{arguments}") do |pipe|
|
39
|
+
IO.popen("#{binary} #{arguments} 2>&1") do |pipe|
|
40
40
|
output = pipe.readlines
|
41
41
|
end
|
42
42
|
$?.exitstatus.should == 0
|
data/spec/lib/mocks.rb
CHANGED
@@ -1,20 +1,20 @@
|
|
1
1
|
module RequestLogAnalyzer::RSpec::Mocks
|
2
2
|
|
3
3
|
def mock_source
|
4
|
-
source =
|
5
|
-
source.stub
|
6
|
-
source.stub
|
7
|
-
source.stub
|
8
|
-
source.stub
|
4
|
+
source = double('RequestLogAnalyzer::Source::Base')
|
5
|
+
source.stub(:file_format).and_return(testing_format)
|
6
|
+
source.stub(:parsed_requests).and_return(2)
|
7
|
+
source.stub(:skipped_requests).and_return(1)
|
8
|
+
source.stub(:parse_lines).and_return(10)
|
9
9
|
|
10
|
-
source.stub
|
11
|
-
source.stub
|
12
|
-
source.stub
|
10
|
+
source.stub(:warning=)
|
11
|
+
source.stub(:progress=)
|
12
|
+
source.stub(:source_changes=)
|
13
13
|
|
14
|
-
source.stub
|
15
|
-
source.stub
|
14
|
+
source.stub(:prepare)
|
15
|
+
source.stub(:finalize)
|
16
16
|
|
17
|
-
source.stub
|
17
|
+
source.stub(:each_request).
|
18
18
|
and_yield(testing_format.request(:field => 'value1')).
|
19
19
|
and_yield(testing_format.request(:field => 'value2'))
|
20
20
|
|
@@ -22,51 +22,51 @@ module RequestLogAnalyzer::RSpec::Mocks
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def mock_io
|
25
|
-
mio =
|
26
|
-
mio.stub
|
27
|
-
mio.stub
|
28
|
-
mio.stub
|
25
|
+
mio = double('IO')
|
26
|
+
mio.stub(:print)
|
27
|
+
mio.stub(:puts)
|
28
|
+
mio.stub(:write)
|
29
29
|
return mio
|
30
30
|
end
|
31
31
|
|
32
32
|
def mock_output
|
33
|
-
output =
|
34
|
-
output.stub
|
35
|
-
output.stub
|
36
|
-
output.stub
|
37
|
-
output.stub
|
38
|
-
output.stub
|
39
|
-
output.stub
|
40
|
-
output.stub
|
41
|
-
output.stub
|
42
|
-
output.stub
|
43
|
-
output.stub
|
44
|
-
output.stub
|
45
|
-
output.stub
|
46
|
-
output.stub
|
47
|
-
output.stub
|
33
|
+
output = double('RequestLogAnalyzer::Output::Base')
|
34
|
+
output.stub(:report_tracker)
|
35
|
+
output.stub(:header)
|
36
|
+
output.stub(:footer)
|
37
|
+
output.stub(:puts)
|
38
|
+
output.stub(:<<)
|
39
|
+
output.stub(:colorize).and_return("Fancy text")
|
40
|
+
output.stub(:link)
|
41
|
+
output.stub(:title)
|
42
|
+
output.stub(:line)
|
43
|
+
output.stub(:with_style)
|
44
|
+
output.stub(:table).and_yield([])
|
45
|
+
output.stub(:io).and_return(mock_io)
|
46
|
+
output.stub(:options).and_return({})
|
47
|
+
output.stub(:slice_results).and_return { |a| a }
|
48
48
|
return output
|
49
49
|
end
|
50
50
|
|
51
51
|
def mock_database(*stubs)
|
52
|
-
database =
|
53
|
-
database.stub
|
54
|
-
database.stub
|
55
|
-
database.stub
|
56
|
-
stubs.each { |s| database.stub
|
52
|
+
database = double('RequestLogAnalyzer::Database')
|
53
|
+
database.stub(:connect)
|
54
|
+
database.stub(:disconnect)
|
55
|
+
database.stub(:connection).and_return(mock_connection)
|
56
|
+
stubs.each { |s| database.stub(s)}
|
57
57
|
return database
|
58
58
|
end
|
59
59
|
|
60
60
|
def mock_connection
|
61
|
-
table_creator =
|
62
|
-
table_creator.stub
|
61
|
+
table_creator = double('ActiveRecord table creator')
|
62
|
+
table_creator.stub(:column)
|
63
63
|
|
64
|
-
connection =
|
65
|
-
connection.stub
|
66
|
-
connection.stub
|
67
|
-
connection.stub
|
68
|
-
connection.stub
|
69
|
-
connection.stub
|
64
|
+
connection = double('ActiveRecord::Base.connection')
|
65
|
+
connection.stub(:add_index)
|
66
|
+
connection.stub(:remove_index)
|
67
|
+
connection.stub(:table_exists?).and_return(false)
|
68
|
+
connection.stub(:create_table).and_yield(table_creator).and_return(true)
|
69
|
+
connection.stub(:table_creator).and_return(table_creator)
|
70
70
|
return connection
|
71
71
|
end
|
72
72
|
end
|
@@ -13,7 +13,7 @@ describe RequestLogAnalyzer::Aggregator::DatabaseInserter do
|
|
13
13
|
before(:each) do
|
14
14
|
@database = mock_database(:create_database_schema!, :drop_database_schema!, :file_format=)
|
15
15
|
@database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser)
|
16
|
-
RequestLogAnalyzer::Database.stub
|
16
|
+
RequestLogAnalyzer::Database.stub(:new).and_return(@database)
|
17
17
|
end
|
18
18
|
|
19
19
|
it 'should establish the database connection' do
|
@@ -4,8 +4,8 @@ describe RequestLogAnalyzer::Controller do
|
|
4
4
|
|
5
5
|
it "should use a custom output generator correctly" do
|
6
6
|
|
7
|
-
mock_output =
|
8
|
-
mock_output.stub
|
7
|
+
mock_output = double('RequestLogAnalyzer::Output::Base')
|
8
|
+
mock_output.stub(:io).and_return(mock_io)
|
9
9
|
mock_output.should_receive(:header)
|
10
10
|
mock_output.should_receive(:footer)
|
11
11
|
|
@@ -17,7 +17,7 @@ describe RequestLogAnalyzer::Controller do
|
|
17
17
|
it "should call aggregators correctly when run" do
|
18
18
|
controller = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
|
19
19
|
|
20
|
-
mock_aggregator =
|
20
|
+
mock_aggregator = double('RequestLogAnalyzer::Aggregator::Base')
|
21
21
|
mock_aggregator.should_receive(:prepare).once.ordered
|
22
22
|
mock_aggregator.should_receive(:aggregate).with(an_instance_of(testing_format.request_class)).twice.ordered
|
23
23
|
mock_aggregator.should_receive(:finalize).once.ordered
|
@@ -30,7 +30,7 @@ describe RequestLogAnalyzer::Controller do
|
|
30
30
|
it "should call filters when run" do
|
31
31
|
controller = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
|
32
32
|
|
33
|
-
mock_filter =
|
33
|
+
mock_filter = double('RequestLogAnalyzer::Filter::Base')
|
34
34
|
mock_filter.should_receive(:filter).twice.and_return(nil)
|
35
35
|
controller.should_receive(:aggregate_request).twice.and_return(nil)
|
36
36
|
|
@@ -11,17 +11,17 @@ describe RequestLogAnalyzer::Database::Base do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
before(:each) do
|
14
|
-
@orm_class =
|
15
|
-
@orm_class.stub
|
16
|
-
@orm_class.stub
|
17
|
-
@orm_class.stub
|
18
|
-
@orm_class.stub
|
14
|
+
@orm_class = double('Line ActiveRecord::Base class')
|
15
|
+
@orm_class.stub("table_name=")
|
16
|
+
@orm_class.stub(:belongs_to)
|
17
|
+
@orm_class.stub(:serialize)
|
18
|
+
@orm_class.stub(:line_definition=)
|
19
19
|
|
20
|
-
RequestLogAnalyzer::Database::Request.stub
|
21
|
-
RequestLogAnalyzer::Database::Source.stub
|
20
|
+
RequestLogAnalyzer::Database::Request.stub(:has_many)
|
21
|
+
RequestLogAnalyzer::Database::Source.stub(:has_many)
|
22
22
|
|
23
23
|
@database = mock_database
|
24
|
-
RequestLogAnalyzer::Database::Base.stub
|
24
|
+
RequestLogAnalyzer::Database::Base.stub(:database).and_return(@database)
|
25
25
|
end
|
26
26
|
|
27
27
|
it "should create a new subclass using the Base class as parent" do
|
@@ -68,17 +68,17 @@ describe RequestLogAnalyzer::Database::Base do
|
|
68
68
|
describe '.subclass_from_table' do
|
69
69
|
before(:each) do
|
70
70
|
|
71
|
-
RequestLogAnalyzer::Database::Request.stub
|
72
|
-
RequestLogAnalyzer::Database::Source.stub
|
71
|
+
RequestLogAnalyzer::Database::Request.stub(:has_many)
|
72
|
+
RequestLogAnalyzer::Database::Source.stub(:has_many)
|
73
73
|
|
74
74
|
@database = mock_database
|
75
|
-
@database.connection.stub
|
76
|
-
RequestLogAnalyzer::Database::Base.stub
|
75
|
+
@database.connection.stub(:table_exists?).and_return(true)
|
76
|
+
RequestLogAnalyzer::Database::Base.stub(:database).and_return(@database)
|
77
77
|
|
78
|
-
@klass =
|
79
|
-
@klass.stub
|
80
|
-
@klass.stub
|
81
|
-
@klass.stub
|
78
|
+
@klass = double('ActiveRecord ORM class')
|
79
|
+
@klass.stub(:column_names).and_return(['id', 'request_id', 'source_id', 'lineno', 'duration'])
|
80
|
+
@klass.stub("table_name=")
|
81
|
+
@klass.stub(:belongs_to)
|
82
82
|
end
|
83
83
|
|
84
84
|
it "should set the table name" do
|
@@ -118,9 +118,9 @@ describe RequestLogAnalyzer::Database::Base do
|
|
118
118
|
|
119
119
|
before(:each) do
|
120
120
|
@database = RequestLogAnalyzer::Database.new
|
121
|
-
@database.stub
|
121
|
+
@database.stub(:connection).and_return(mock_connection)
|
122
122
|
@klass = @database.load_activerecord_class(@line_definition)
|
123
|
-
@klass.stub
|
123
|
+
@klass.stub(:table_exists?).and_return(false)
|
124
124
|
end
|
125
125
|
|
126
126
|
after(:each) do
|
@@ -134,7 +134,7 @@ describe RequestLogAnalyzer::Database::Base do
|
|
134
134
|
end
|
135
135
|
|
136
136
|
it "should not create a table based on the line type name if it already exists" do
|
137
|
-
@klass.stub
|
137
|
+
@klass.stub(:table_exists?).and_return(true)
|
138
138
|
@database.connection.should_not_receive(:create_table).with(:test_lines)
|
139
139
|
@klass.create_table!
|
140
140
|
end
|
@@ -51,11 +51,11 @@ describe RequestLogAnalyzer::Database do
|
|
51
51
|
before(:each) do
|
52
52
|
@database = RequestLogAnalyzer::Database.new
|
53
53
|
@database.file_format = testing_format
|
54
|
-
@database.stub
|
54
|
+
@database.stub(:connection).and_return(mock_connection)
|
55
55
|
|
56
56
|
# Stub the expected method calls for the preparation, these will be tested separately
|
57
57
|
@mock_class = Class.new(RequestLogAnalyzer::Database::Base)
|
58
|
-
@mock_class.stub
|
58
|
+
@mock_class.stub(:create_table!)
|
59
59
|
end
|
60
60
|
|
61
61
|
after(:each) { @database.remove_orm_classes! }
|
@@ -92,16 +92,16 @@ describe RequestLogAnalyzer::Database do
|
|
92
92
|
before(:each) do
|
93
93
|
@database = RequestLogAnalyzer::Database.new
|
94
94
|
@connection = mock_connection
|
95
|
-
@database.stub
|
95
|
+
@database.stub(:connection).and_return(@connection)
|
96
96
|
|
97
97
|
# Mock the has_many method of the defaukt ORM classes
|
98
|
-
RequestLogAnalyzer::Database::Request.stub
|
99
|
-
RequestLogAnalyzer::Database::Source.stub
|
98
|
+
RequestLogAnalyzer::Database::Request.stub(:has_many)
|
99
|
+
RequestLogAnalyzer::Database::Source.stub(:has_many)
|
100
100
|
|
101
101
|
@mock_class = Class.new(RequestLogAnalyzer::Database::Base)
|
102
102
|
|
103
|
-
RequestLogAnalyzer::Database::Base.stub
|
104
|
-
RequestLogAnalyzer::Database::Base.stub
|
103
|
+
RequestLogAnalyzer::Database::Base.stub(:subclass_from_table).and_return(@mock_class)
|
104
|
+
RequestLogAnalyzer::Database::Base.stub(:subclass_from_line_definition).and_return(@mock_class)
|
105
105
|
end
|
106
106
|
|
107
107
|
after(:each) { @database.remove_orm_classes! }
|
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
4
|
-
|
4
|
+
|
5
5
|
subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
|
6
6
|
|
7
7
|
it { should be_well_formed }
|
@@ -12,55 +12,77 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
|
|
12
12
|
|
13
13
|
let(:sample_get) { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
|
14
14
|
let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
|
15
|
-
|
15
|
+
let(:sample_2013) { 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac testbucket [17/Sep/2013:15:53:53 +0000] 87.193.165.87 - A9774F82F053FACE REST.GET.OBJECT somefile.txt "GET /testbucket/somefile.txt HTTP/1.1" 304 - - 1071 9 - "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36" -' }
|
16
|
+
|
16
17
|
describe '#parse_line' do
|
17
|
-
it {
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
18
|
+
it {
|
19
|
+
should parse_line(sample_get, 'a GET line').and_capture(
|
20
|
+
:bucket_owner => '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
|
21
|
+
:bucket => 'sample-bucket',
|
22
|
+
:timestamp => 20061006014214,
|
23
|
+
:remote_ip => '207.171.172.6',
|
24
|
+
:key => 'object.png',
|
25
|
+
:operation => 'REST.GET.OBJECT',
|
26
|
+
:requester => '65a011a29cdf8ec533ec3d1ccaae921c',
|
27
|
+
:request_id => 'C980091AD89C936A',
|
28
|
+
:request_uri => 'GET /sample-bucket/object.png HTTP/1.1',
|
29
|
+
:error_code => nil,
|
30
|
+
:http_status => 200,
|
31
|
+
:total_time => 0.988,
|
32
|
+
:turnaround_time => 0.987,
|
33
|
+
:bytes_sent => 1243,
|
34
|
+
:object_size => 1243,
|
35
|
+
:user_agent => 'aranhabot',
|
36
|
+
:referer => nil)
|
37
|
+
}
|
38
|
+
|
39
|
+
it { should parse_line(sample_2013, '2013 sample').and_capture(
|
40
|
+
:bucket_owner => 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac',
|
41
|
+
:bucket => 'testbucket',
|
42
|
+
:timestamp => 20130917155353,
|
43
|
+
:remote_ip => '87.193.165.87',
|
44
|
+
:requester => '-',
|
45
|
+
:request_id => 'A9774F82F053FACE',
|
46
|
+
:operation => 'REST.GET.OBJECT',
|
47
|
+
:key => 'somefile.txt',
|
48
|
+
:request_uri => "GET /testbucket/somefile.txt HTTP/1.1",
|
49
|
+
:http_status => 304,
|
50
|
+
:error_code => nil,
|
51
|
+
:bytes_sent => 0,
|
52
|
+
:object_size => 1071,
|
53
|
+
:total_time => 0.009,
|
54
|
+
:turnaround_time => 0,
|
55
|
+
:referer => "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US",
|
56
|
+
:user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36")
|
35
57
|
}
|
36
58
|
|
37
|
-
it {
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
59
|
+
it {should parse_line(sample_copy, 'a COPY line').and_capture(
|
60
|
+
:bucket_owner => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
61
|
+
:bucket => 'testbucket',
|
62
|
+
:timestamp => 20100303230459,
|
63
|
+
:remote_ip => '174.119.31.76',
|
64
|
+
:key => 'files/image.png',
|
65
|
+
:operation => 'REST.COPY.OBJECT',
|
66
|
+
:requester => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
|
67
|
+
:request_id => 'ACCC34B843C87BC9',
|
68
|
+
:request_uri => 'PUT /files/image.png HTTP/1.1',
|
69
|
+
:error_code => nil,
|
70
|
+
:http_status => 200,
|
71
|
+
:total_time => 0.365,
|
72
|
+
:turnaround_time => 0.319,
|
73
|
+
:bytes_sent => 234,
|
74
|
+
:object_size => 65957,
|
75
|
+
:user_agent => '',
|
76
|
+
:referer => nil)
|
55
77
|
}
|
56
78
|
|
57
79
|
it { should_not parse_line('nonsense', 'a nonsense line') }
|
58
80
|
end
|
59
|
-
|
81
|
+
|
60
82
|
describe '#parse_io' do
|
61
83
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
62
84
|
let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
|
63
|
-
|
85
|
+
|
64
86
|
it "should parse requests correctly and not generate warnings" do
|
65
87
|
log_parser.should_receive(:handle_request).twice
|
66
88
|
log_parser.should_not_receive(:warn)
|
@@ -26,18 +26,18 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
26
26
|
line_definition.captures?(:duration).should be_true
|
27
27
|
end
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
describe '.access_line_definition' do
|
31
31
|
it "should parse values in microseconds when no argument is given to %D" do
|
32
32
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
|
33
33
|
format.should parse_line('12345').and_capture(:duration => 0.012345)
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
it "should parse values in microseconds when micro is given as argument to %D" do
|
37
37
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
|
38
38
|
format.should parse_line('12345').and_capture(:duration => 0.012345)
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
it "should parse values in microseconds when micro is given as argument to %D" do
|
42
42
|
format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
|
43
43
|
format.should parse_line('12345').and_capture(:duration => 12.345)
|
@@ -53,36 +53,50 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
53
53
|
it { should have(8).report_trackers }
|
54
54
|
end
|
55
55
|
|
56
|
+
context '"vhost_combined" access log parsing' do
|
57
|
+
subject { RequestLogAnalyzer::FileFormat.load(:apache, :vhost_combined) }
|
58
|
+
describe '#parse_line' do
|
59
|
+
let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683'}
|
60
|
+
|
61
|
+
it { should parse_line(sample1, 'a sample line').and_capture(
|
62
|
+
:remote_host => '72.204.80.86', :remote_logname => nil, :user => nil,
|
63
|
+
:vhost => 'psi-equipment.od1.vtiger.com', :duration => 1.0,
|
64
|
+
:timestamp => 20131019154154, :http_status => 304, :http_method => 'GET',
|
65
|
+
:http_version => '1.1', :bytes_sent => 0)
|
66
|
+
}
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
56
70
|
context '"Common" access log parsing' do
|
57
|
-
subject { RequestLogAnalyzer::FileFormat.load(:apache, :common) }
|
58
|
-
|
71
|
+
subject { RequestLogAnalyzer::FileFormat.load(:apache, :common) }
|
72
|
+
|
59
73
|
it { should be_well_formed }
|
60
74
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent) }
|
61
75
|
it { should have(6).report_trackers }
|
62
|
-
|
76
|
+
|
63
77
|
describe '#parse_line' do
|
64
|
-
|
78
|
+
|
65
79
|
let(:sample1) { '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223' }
|
66
80
|
let(:sample2) { '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414' }
|
67
|
-
|
81
|
+
|
68
82
|
it { should parse_line(sample1, 'a sample line').and_capture(
|
69
83
|
:remote_host => '1.129.119.13', :remote_logname => nil, :user => nil,
|
70
84
|
:timestamp => 20090908075409, :http_status => 200, :http_method => 'GET',
|
71
85
|
:http_version => '1.0', :bytes_sent => 8223)
|
72
86
|
}
|
73
|
-
|
87
|
+
|
74
88
|
it { should parse_line(sample2, 'another sample line').and_capture(
|
75
89
|
:remote_host => '1.82.235.29', :remote_logname => nil, :user => nil,
|
76
90
|
:timestamp => 20090908075405, :http_status => 200, :http_method => 'GET',
|
77
91
|
:http_version => '1.1', :bytes_sent => 23414)
|
78
92
|
}
|
79
|
-
|
93
|
+
|
80
94
|
it { should_not parse_line('nonsense', 'a nonsense line')}
|
81
95
|
end
|
82
|
-
|
96
|
+
|
83
97
|
describe '#parse_io' do
|
84
98
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
85
|
-
|
99
|
+
|
86
100
|
it "should parse a log snippet successfully without warnings" do
|
87
101
|
log_parser.should_receive(:handle_request).exactly(10).times
|
88
102
|
log_parser.should_not_receive(:warn)
|
@@ -92,36 +106,36 @@ describe RequestLogAnalyzer::FileFormat::Apache do
|
|
92
106
|
end
|
93
107
|
|
94
108
|
context '"Combined" access log parsing' do
|
95
|
-
subject { RequestLogAnalyzer::FileFormat.load(:apache, :combined) }
|
96
|
-
|
109
|
+
subject { RequestLogAnalyzer::FileFormat.load(:apache, :combined) }
|
110
|
+
|
97
111
|
it { should be_well_formed }
|
98
112
|
it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent, :referer, :user_agent) }
|
99
113
|
it { should have(8).report_trackers }
|
100
|
-
|
114
|
+
|
101
115
|
describe '#parse_line' do
|
102
116
|
let(:sample1) { '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"' }
|
103
117
|
let(:sample2) { '0:0:0:0:0:0:0:1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"' }
|
104
|
-
|
118
|
+
|
105
119
|
it { should parse_line(sample1, 'with IPv4 address').and_capture(
|
106
120
|
:remote_host => '69.41.0.45', :remote_logname => nil, :user => nil,
|
107
121
|
:timestamp => 20090902120240, :http_status => 404, :http_method => 'GET',
|
108
122
|
:http_version => '1.1', :bytes_sent => 209, :referer => nil,
|
109
123
|
:user_agent => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
|
110
124
|
}
|
111
|
-
|
125
|
+
|
112
126
|
it { should parse_line(sample2, 'with IPv6 address').and_capture(
|
113
127
|
:remote_host => '0:0:0:0:0:0:0:1', :remote_logname => nil, :user => nil,
|
114
128
|
:timestamp => 20090902050833, :http_status => 200, :http_method => 'GET',
|
115
129
|
:http_version => '1.1', :bytes_sent => 30, :referer => nil,
|
116
130
|
:user_agent => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
|
117
131
|
}
|
118
|
-
|
132
|
+
|
119
133
|
it { should_not parse_line('nonsense', 'a nonsense line')}
|
120
134
|
end
|
121
|
-
|
135
|
+
|
122
136
|
describe '#parse_io' do
|
123
137
|
let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
|
124
|
-
|
138
|
+
|
125
139
|
it "should parse a log snippet successfully without warnings" do
|
126
140
|
log_parser.should_receive(:handle_request).exactly(5).times
|
127
141
|
log_parser.should_not_receive(:warn)
|
@@ -31,7 +31,7 @@ describe RequestLogAnalyzer::LineDefinition do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
describe '#convert_captured_values' do
|
34
|
-
let(:request) {
|
34
|
+
let(:request) { double('request', :convert_value => 'foo') }
|
35
35
|
|
36
36
|
it "should call convert_value for every captured value" do
|
37
37
|
request.should_receive(:convert_value).twice
|
@@ -50,8 +50,8 @@ describe RequestLogAnalyzer::LineDefinition do
|
|
50
50
|
}
|
51
51
|
|
52
52
|
before do
|
53
|
-
request.stub
|
54
|
-
request.stub
|
53
|
+
request.stub(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
|
54
|
+
request.stub(:convert_value).with('baz', anything).and_return('foo')
|
55
55
|
end
|
56
56
|
|
57
57
|
it "should call Request#convert_value for the initial hash and the value in the hash" do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: request-log-analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.12.
|
4
|
+
version: 1.12.10
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-10-20 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rake
|
@@ -35,7 +35,7 @@ dependencies:
|
|
35
35
|
requirements:
|
36
36
|
- - ~>
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version: '2.
|
38
|
+
version: '2.14'
|
39
39
|
type: :development
|
40
40
|
prerelease: false
|
41
41
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ~>
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: '2.
|
46
|
+
version: '2.14'
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: activerecord
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -131,6 +131,9 @@ files:
|
|
131
131
|
- .travis.yml
|
132
132
|
- DESIGN.rdoc
|
133
133
|
- Gemfile
|
134
|
+
- Gemfile.activerecord2
|
135
|
+
- Gemfile.activerecord3
|
136
|
+
- Gemfile.activerecord4
|
134
137
|
- LICENSE
|
135
138
|
- README.rdoc
|
136
139
|
- Rakefile
|
@@ -276,7 +279,8 @@ files:
|
|
276
279
|
- spec/unit/tracker/traffic_tracker_spec.rb
|
277
280
|
- tasks/request_log_analyzer.rake
|
278
281
|
homepage: http://railsdoctors.com
|
279
|
-
licenses:
|
282
|
+
licenses:
|
283
|
+
- MIT
|
280
284
|
post_install_message:
|
281
285
|
rdoc_options:
|
282
286
|
- --title
|
@@ -295,7 +299,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
295
299
|
version: '0'
|
296
300
|
segments:
|
297
301
|
- 0
|
298
|
-
hash:
|
302
|
+
hash: -262165736928167385
|
299
303
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
300
304
|
none: false
|
301
305
|
requirements:
|
@@ -304,7 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
304
308
|
version: '0'
|
305
309
|
segments:
|
306
310
|
- 0
|
307
|
-
hash:
|
311
|
+
hash: -262165736928167385
|
308
312
|
requirements:
|
309
313
|
- To use the database inserter, ActiveRecord and an appropriate database adapter are
|
310
314
|
required.
|