request-log-analyzer 1.12.9 → 1.12.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.travis.yml CHANGED
@@ -14,10 +14,23 @@ rvm:
14
14
  - jruby-head
15
15
  - rbx-18mode
16
16
  - rbx-19mode
17
+ gemfile:
18
+ - Gemfile.activerecord2
19
+ - Gemfile.activerecord3
20
+ - Gemfile.activerecord4
17
21
  matrix:
18
22
  allow_failures:
19
23
  - rvm: jruby-head
20
24
  - rvm: ruby-head
25
+ exclude:
26
+ - gemfile: Gemfile.activerecord4
27
+ rvm: 1.8.7
28
+ - gemfile: Gemfile.activerecord4
29
+ rvm: ree
30
+ - gemfile: Gemfile.activerecord4
31
+ rvm: jruby-18mode
32
+ - gemfile: Gemfile.activerecord4
33
+ rvm: rbx-18mode
21
34
  notifications:
22
35
  email:
23
36
  - info@railsdoctors.com
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ gem 'activerecord', '~> 2.1'
5
+ gem 'activerecord-mysql2-adapter', :platforms => [:ruby]
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ gem 'activerecord', '~> 3.0'
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ gem 'activerecord', '~> 4.0'
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2008-2009 Willem van Bergen & Bart ten Brinke
1
+ Copyright (c) 2008-2013 Willem van Bergen & Bart ten Brinke
2
2
 
3
3
  Permission is hereby granted, free of charge, to any person obtaining
4
4
  a copy of this software and associated documentation files (the
data/README.rdoc CHANGED
@@ -47,5 +47,4 @@ Bergen (willem@railsdoctors.com) or Bart ten Brinke (bart@railsdoctors.com).
47
47
 
48
48
  * Project wiki at GitHub: http://github.com/wvanbergen/request-log-analyzer/wiki
49
49
  * Issue tracker at GitHub: http://github.com/wvanbergen/request-log-analyzer/issues
50
- * railsdoctors homepage: http://railsdoctors.com
51
- * wvanbergen's blog posts: http://techblog.floorplanner.com/tag/request-log-analyzer
50
+ * railsdoctors homepage: http://railsdoctors.com
@@ -5,14 +5,14 @@ module RequestLogAnalyzer::FileFormat
5
5
  # Access logs are disabled by default on Amazon S3. To enable logging, see
6
6
  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
7
7
  class AmazonS3 < Base
8
-
8
+
9
9
  extend CommonRegularExpressions
10
10
 
11
11
  line_definition :access do |line|
12
12
  line.header = true
13
13
  line.footer = true
14
- line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]*)" "([^"]*)"/
15
-
14
+ line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) ([^\ ]+) (\d+) (\d+) ([^\ ]+) "([^"]*)" "([^"]*)"/
15
+
16
16
  line.capture(:bucket_owner)
17
17
  line.capture(:bucket)
18
18
  line.capture(:timestamp).as(:timestamp)
@@ -20,6 +20,7 @@ module RequestLogAnalyzer::FileFormat
20
20
  LOG_FORMAT_DEFAULTS = {
21
21
  :common => '%h %l %u %t "%r" %>s %b',
22
22
  :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
23
+ :vhost_combined => '%h %l %v %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T/%D',
23
24
  :nginx => '%a %t %h %u "%r" %>s %b',
24
25
  :rack => '%h %l %u %t "%r" %>s %b %T',
25
26
  :referer => '%{Referer}i -> %U',
@@ -32,6 +33,7 @@ module RequestLogAnalyzer::FileFormat
32
33
  # A hash that defines how the log format directives should be parsed.
33
34
  LOG_DIRECTIVES = {
34
35
  '%' => { nil => { :regexp => '%', :captures => [] } },
36
+ 'v' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :vhost, :type => :string}] } },
35
37
  'h' => { nil => { :regexp => "(#{hostname_or_ip_address})", :captures => [{:name => :remote_host, :type => :string}] } },
36
38
  'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
37
39
  'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
@@ -41,7 +43,7 @@ module RequestLogAnalyzer::FileFormat
41
43
  'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
42
44
  },
43
45
  'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
44
- 'T' => { nil => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
46
+ 'T' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
45
47
  't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
46
48
  's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
47
49
  'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
@@ -66,12 +68,11 @@ module RequestLogAnalyzer::FileFormat
66
68
  def self.access_line_definition(format_string)
67
69
  format_string ||= :common
68
70
  format_string = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
69
-
70
-
71
+
71
72
  line_regexp = ''
72
73
  captures = []
73
74
  format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
74
-
75
+
75
76
  line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
76
77
 
77
78
  if variable
@@ -87,7 +88,7 @@ module RequestLogAnalyzer::FileFormat
87
88
  end
88
89
  end
89
90
  end
90
-
91
+
91
92
  # Return a new line definition object
92
93
  return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
93
94
  :captures => captures, :header => true, :footer => true)
@@ -1,3 +1,3 @@
1
1
  module RequestLogAnalyzer
2
- VERSION = "1.12.9"
2
+ VERSION = "1.12.10"
3
3
  end
@@ -12,6 +12,7 @@ Gem::Specification.new do |gem|
12
12
  gem.authors = ['Willem van Bergen', 'Bart ten Brinke']
13
13
  gem.email = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
14
14
  gem.homepage = 'http://railsdoctors.com'
15
+ gem.license = "MIT"
15
16
 
16
17
  gem.summary = "A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers"
17
18
  gem.description = <<-eos
@@ -29,7 +30,7 @@ Gem::Specification.new do |gem|
29
30
 
30
31
  gem.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required."
31
32
  gem.add_development_dependency('rake')
32
- gem.add_development_dependency('rspec', '~> 2.13')
33
+ gem.add_development_dependency('rspec', '~> 2.14')
33
34
  gem.add_development_dependency('activerecord')
34
35
  if defined?(JRUBY_VERSION)
35
36
  gem.add_development_dependency('jdbc-sqlite3')
data/spec/lib/helpers.rb CHANGED
@@ -36,7 +36,7 @@ module RequestLogAnalyzer::RSpec::Helpers
36
36
  arguments = arguments.join(' ') if arguments.kind_of?(Array)
37
37
 
38
38
  output = []
39
- IO.popen("#{binary} #{arguments}") do |pipe|
39
+ IO.popen("#{binary} #{arguments} 2>&1") do |pipe|
40
40
  output = pipe.readlines
41
41
  end
42
42
  $?.exitstatus.should == 0
data/spec/lib/mocks.rb CHANGED
@@ -1,20 +1,20 @@
1
1
  module RequestLogAnalyzer::RSpec::Mocks
2
2
 
3
3
  def mock_source
4
- source = mock('RequestLogAnalyzer::Source::Base')
5
- source.stub!(:file_format).and_return(testing_format)
6
- source.stub!(:parsed_requests).and_return(2)
7
- source.stub!(:skipped_requests).and_return(1)
8
- source.stub!(:parse_lines).and_return(10)
4
+ source = double('RequestLogAnalyzer::Source::Base')
5
+ source.stub(:file_format).and_return(testing_format)
6
+ source.stub(:parsed_requests).and_return(2)
7
+ source.stub(:skipped_requests).and_return(1)
8
+ source.stub(:parse_lines).and_return(10)
9
9
 
10
- source.stub!(:warning=)
11
- source.stub!(:progress=)
12
- source.stub!(:source_changes=)
10
+ source.stub(:warning=)
11
+ source.stub(:progress=)
12
+ source.stub(:source_changes=)
13
13
 
14
- source.stub!(:prepare)
15
- source.stub!(:finalize)
14
+ source.stub(:prepare)
15
+ source.stub(:finalize)
16
16
 
17
- source.stub!(:each_request).
17
+ source.stub(:each_request).
18
18
  and_yield(testing_format.request(:field => 'value1')).
19
19
  and_yield(testing_format.request(:field => 'value2'))
20
20
 
@@ -22,51 +22,51 @@ module RequestLogAnalyzer::RSpec::Mocks
22
22
  end
23
23
 
24
24
  def mock_io
25
- mio = mock('IO')
26
- mio.stub!(:print)
27
- mio.stub!(:puts)
28
- mio.stub!(:write)
25
+ mio = double('IO')
26
+ mio.stub(:print)
27
+ mio.stub(:puts)
28
+ mio.stub(:write)
29
29
  return mio
30
30
  end
31
31
 
32
32
  def mock_output
33
- output = mock('RequestLogAnalyzer::Output::Base')
34
- output.stub!(:report_tracker)
35
- output.stub!(:header)
36
- output.stub!(:footer)
37
- output.stub!(:puts)
38
- output.stub!(:<<)
39
- output.stub!(:colorize).and_return("Fancy text")
40
- output.stub!(:link)
41
- output.stub!(:title)
42
- output.stub!(:line)
43
- output.stub!(:with_style)
44
- output.stub!(:table).and_yield([])
45
- output.stub!(:io).and_return(mock_io)
46
- output.stub!(:options).and_return({})
47
- output.stub!(:slice_results).and_return { |a| a }
33
+ output = double('RequestLogAnalyzer::Output::Base')
34
+ output.stub(:report_tracker)
35
+ output.stub(:header)
36
+ output.stub(:footer)
37
+ output.stub(:puts)
38
+ output.stub(:<<)
39
+ output.stub(:colorize).and_return("Fancy text")
40
+ output.stub(:link)
41
+ output.stub(:title)
42
+ output.stub(:line)
43
+ output.stub(:with_style)
44
+ output.stub(:table).and_yield([])
45
+ output.stub(:io).and_return(mock_io)
46
+ output.stub(:options).and_return({})
47
+ output.stub(:slice_results).and_return { |a| a }
48
48
  return output
49
49
  end
50
50
 
51
51
  def mock_database(*stubs)
52
- database = mock('RequestLogAnalyzer::Database')
53
- database.stub!(:connect)
54
- database.stub!(:disconnect)
55
- database.stub!(:connection).and_return(mock_connection)
56
- stubs.each { |s| database.stub!(s)}
52
+ database = double('RequestLogAnalyzer::Database')
53
+ database.stub(:connect)
54
+ database.stub(:disconnect)
55
+ database.stub(:connection).and_return(mock_connection)
56
+ stubs.each { |s| database.stub(s)}
57
57
  return database
58
58
  end
59
59
 
60
60
  def mock_connection
61
- table_creator = mock('ActiveRecord table creator')
62
- table_creator.stub!(:column)
61
+ table_creator = double('ActiveRecord table creator')
62
+ table_creator.stub(:column)
63
63
 
64
- connection = mock('ActiveRecord::Base.connection')
65
- connection.stub!(:add_index)
66
- connection.stub!(:remove_index)
67
- connection.stub!(:table_exists?).and_return(false)
68
- connection.stub!(:create_table).and_yield(table_creator).and_return(true)
69
- connection.stub!(:table_creator).and_return(table_creator)
64
+ connection = double('ActiveRecord::Base.connection')
65
+ connection.stub(:add_index)
66
+ connection.stub(:remove_index)
67
+ connection.stub(:table_exists?).and_return(false)
68
+ connection.stub(:create_table).and_yield(table_creator).and_return(true)
69
+ connection.stub(:table_creator).and_return(table_creator)
70
70
  return connection
71
71
  end
72
72
  end
@@ -13,7 +13,7 @@ describe RequestLogAnalyzer::Aggregator::DatabaseInserter do
13
13
  before(:each) do
14
14
  @database = mock_database(:create_database_schema!, :drop_database_schema!, :file_format=)
15
15
  @database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser)
16
- RequestLogAnalyzer::Database.stub!(:new).and_return(@database)
16
+ RequestLogAnalyzer::Database.stub(:new).and_return(@database)
17
17
  end
18
18
 
19
19
  it 'should establish the database connection' do
@@ -4,8 +4,8 @@ describe RequestLogAnalyzer::Controller do
4
4
 
5
5
  it "should use a custom output generator correctly" do
6
6
 
7
- mock_output = mock('RequestLogAnalyzer::Output::Base')
8
- mock_output.stub!(:io).and_return(mock_io)
7
+ mock_output = double('RequestLogAnalyzer::Output::Base')
8
+ mock_output.stub(:io).and_return(mock_io)
9
9
  mock_output.should_receive(:header)
10
10
  mock_output.should_receive(:footer)
11
11
 
@@ -17,7 +17,7 @@ describe RequestLogAnalyzer::Controller do
17
17
  it "should call aggregators correctly when run" do
18
18
  controller = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
19
19
 
20
- mock_aggregator = mock('RequestLogAnalyzer::Aggregator::Base')
20
+ mock_aggregator = double('RequestLogAnalyzer::Aggregator::Base')
21
21
  mock_aggregator.should_receive(:prepare).once.ordered
22
22
  mock_aggregator.should_receive(:aggregate).with(an_instance_of(testing_format.request_class)).twice.ordered
23
23
  mock_aggregator.should_receive(:finalize).once.ordered
@@ -30,7 +30,7 @@ describe RequestLogAnalyzer::Controller do
30
30
  it "should call filters when run" do
31
31
  controller = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
32
32
 
33
- mock_filter = mock('RequestLogAnalyzer::Filter::Base')
33
+ mock_filter = double('RequestLogAnalyzer::Filter::Base')
34
34
  mock_filter.should_receive(:filter).twice.and_return(nil)
35
35
  controller.should_receive(:aggregate_request).twice.and_return(nil)
36
36
 
@@ -11,17 +11,17 @@ describe RequestLogAnalyzer::Database::Base do
11
11
  end
12
12
 
13
13
  before(:each) do
14
- @orm_class = mock('Line ActiveRecord::Base class')
15
- @orm_class.stub!("table_name=")
16
- @orm_class.stub!(:belongs_to)
17
- @orm_class.stub!(:serialize)
18
- @orm_class.stub!(:line_definition=)
14
+ @orm_class = double('Line ActiveRecord::Base class')
15
+ @orm_class.stub("table_name=")
16
+ @orm_class.stub(:belongs_to)
17
+ @orm_class.stub(:serialize)
18
+ @orm_class.stub(:line_definition=)
19
19
 
20
- RequestLogAnalyzer::Database::Request.stub!(:has_many)
21
- RequestLogAnalyzer::Database::Source.stub!(:has_many)
20
+ RequestLogAnalyzer::Database::Request.stub(:has_many)
21
+ RequestLogAnalyzer::Database::Source.stub(:has_many)
22
22
 
23
23
  @database = mock_database
24
- RequestLogAnalyzer::Database::Base.stub!(:database).and_return(@database)
24
+ RequestLogAnalyzer::Database::Base.stub(:database).and_return(@database)
25
25
  end
26
26
 
27
27
  it "should create a new subclass using the Base class as parent" do
@@ -68,17 +68,17 @@ describe RequestLogAnalyzer::Database::Base do
68
68
  describe '.subclass_from_table' do
69
69
  before(:each) do
70
70
 
71
- RequestLogAnalyzer::Database::Request.stub!(:has_many)
72
- RequestLogAnalyzer::Database::Source.stub!(:has_many)
71
+ RequestLogAnalyzer::Database::Request.stub(:has_many)
72
+ RequestLogAnalyzer::Database::Source.stub(:has_many)
73
73
 
74
74
  @database = mock_database
75
- @database.connection.stub!(:table_exists?).and_return(true)
76
- RequestLogAnalyzer::Database::Base.stub!(:database).and_return(@database)
75
+ @database.connection.stub(:table_exists?).and_return(true)
76
+ RequestLogAnalyzer::Database::Base.stub(:database).and_return(@database)
77
77
 
78
- @klass = mock('ActiveRecord ORM class')
79
- @klass.stub!(:column_names).and_return(['id', 'request_id', 'source_id', 'lineno', 'duration'])
80
- @klass.stub!("table_name=")
81
- @klass.stub!(:belongs_to)
78
+ @klass = double('ActiveRecord ORM class')
79
+ @klass.stub(:column_names).and_return(['id', 'request_id', 'source_id', 'lineno', 'duration'])
80
+ @klass.stub("table_name=")
81
+ @klass.stub(:belongs_to)
82
82
  end
83
83
 
84
84
  it "should set the table name" do
@@ -118,9 +118,9 @@ describe RequestLogAnalyzer::Database::Base do
118
118
 
119
119
  before(:each) do
120
120
  @database = RequestLogAnalyzer::Database.new
121
- @database.stub!(:connection).and_return(mock_connection)
121
+ @database.stub(:connection).and_return(mock_connection)
122
122
  @klass = @database.load_activerecord_class(@line_definition)
123
- @klass.stub!(:table_exists?).and_return(false)
123
+ @klass.stub(:table_exists?).and_return(false)
124
124
  end
125
125
 
126
126
  after(:each) do
@@ -134,7 +134,7 @@ describe RequestLogAnalyzer::Database::Base do
134
134
  end
135
135
 
136
136
  it "should not create a table based on the line type name if it already exists" do
137
- @klass.stub!(:table_exists?).and_return(true)
137
+ @klass.stub(:table_exists?).and_return(true)
138
138
  @database.connection.should_not_receive(:create_table).with(:test_lines)
139
139
  @klass.create_table!
140
140
  end
@@ -51,11 +51,11 @@ describe RequestLogAnalyzer::Database do
51
51
  before(:each) do
52
52
  @database = RequestLogAnalyzer::Database.new
53
53
  @database.file_format = testing_format
54
- @database.stub!(:connection).and_return(mock_connection)
54
+ @database.stub(:connection).and_return(mock_connection)
55
55
 
56
56
  # Stub the expected method calls for the preparation, these will be tested separately
57
57
  @mock_class = Class.new(RequestLogAnalyzer::Database::Base)
58
- @mock_class.stub!(:create_table!)
58
+ @mock_class.stub(:create_table!)
59
59
  end
60
60
 
61
61
  after(:each) { @database.remove_orm_classes! }
@@ -92,16 +92,16 @@ describe RequestLogAnalyzer::Database do
92
92
  before(:each) do
93
93
  @database = RequestLogAnalyzer::Database.new
94
94
  @connection = mock_connection
95
- @database.stub!(:connection).and_return(@connection)
95
+ @database.stub(:connection).and_return(@connection)
96
96
 
97
97
  # Mock the has_many method of the defaukt ORM classes
98
- RequestLogAnalyzer::Database::Request.stub!(:has_many)
99
- RequestLogAnalyzer::Database::Source.stub!(:has_many)
98
+ RequestLogAnalyzer::Database::Request.stub(:has_many)
99
+ RequestLogAnalyzer::Database::Source.stub(:has_many)
100
100
 
101
101
  @mock_class = Class.new(RequestLogAnalyzer::Database::Base)
102
102
 
103
- RequestLogAnalyzer::Database::Base.stub!(:subclass_from_table).and_return(@mock_class)
104
- RequestLogAnalyzer::Database::Base.stub!(:subclass_from_line_definition).and_return(@mock_class)
103
+ RequestLogAnalyzer::Database::Base.stub(:subclass_from_table).and_return(@mock_class)
104
+ RequestLogAnalyzer::Database::Base.stub(:subclass_from_line_definition).and_return(@mock_class)
105
105
  end
106
106
 
107
107
  after(:each) { @database.remove_orm_classes! }
@@ -1,7 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe RequestLogAnalyzer::FileFormat::AmazonS3 do
4
-
4
+
5
5
  subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
6
6
 
7
7
  it { should be_well_formed }
@@ -12,55 +12,77 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
12
12
 
13
13
  let(:sample_get) { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
14
14
  let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
15
-
15
+ let(:sample_2013) { 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac testbucket [17/Sep/2013:15:53:53 +0000] 87.193.165.87 - A9774F82F053FACE REST.GET.OBJECT somefile.txt "GET /testbucket/somefile.txt HTTP/1.1" 304 - - 1071 9 - "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36" -' }
16
+
16
17
  describe '#parse_line' do
17
- it { should parse_line(sample_get, 'a GET line').and_capture(
18
- :bucket_owner => '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
19
- :bucket => 'sample-bucket',
20
- :timestamp => 20061006014214,
21
- :remote_ip => '207.171.172.6',
22
- :key => 'object.png',
23
- :operation => 'REST.GET.OBJECT',
24
- :requester => '65a011a29cdf8ec533ec3d1ccaae921c',
25
- :request_id => 'C980091AD89C936A',
26
- :request_uri => 'GET /sample-bucket/object.png HTTP/1.1',
27
- :error_code => nil,
28
- :http_status => 200,
29
- :total_time => 0.988,
30
- :turnaround_time => 0.987,
31
- :bytes_sent => 1243,
32
- :object_size => 1243,
33
- :user_agent => 'aranhabot',
34
- :referer => nil)
18
+ it {
19
+ should parse_line(sample_get, 'a GET line').and_capture(
20
+ :bucket_owner => '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
21
+ :bucket => 'sample-bucket',
22
+ :timestamp => 20061006014214,
23
+ :remote_ip => '207.171.172.6',
24
+ :key => 'object.png',
25
+ :operation => 'REST.GET.OBJECT',
26
+ :requester => '65a011a29cdf8ec533ec3d1ccaae921c',
27
+ :request_id => 'C980091AD89C936A',
28
+ :request_uri => 'GET /sample-bucket/object.png HTTP/1.1',
29
+ :error_code => nil,
30
+ :http_status => 200,
31
+ :total_time => 0.988,
32
+ :turnaround_time => 0.987,
33
+ :bytes_sent => 1243,
34
+ :object_size => 1243,
35
+ :user_agent => 'aranhabot',
36
+ :referer => nil)
37
+ }
38
+
39
+ it { should parse_line(sample_2013, '2013 sample').and_capture(
40
+ :bucket_owner => 'ccc30d58fff2fad852150fa6f9e7bd14b5f6d2ea83f2deec4a610bf0f0a8f7ac',
41
+ :bucket => 'testbucket',
42
+ :timestamp => 20130917155353,
43
+ :remote_ip => '87.193.165.87',
44
+ :requester => '-',
45
+ :request_id => 'A9774F82F053FACE',
46
+ :operation => 'REST.GET.OBJECT',
47
+ :key => 'somefile.txt',
48
+ :request_uri => "GET /testbucket/somefile.txt HTTP/1.1",
49
+ :http_status => 304,
50
+ :error_code => nil,
51
+ :bytes_sent => 0,
52
+ :object_size => 1071,
53
+ :total_time => 0.009,
54
+ :turnaround_time => 0,
55
+ :referer => "https://s3-console-us-standard.console.aws.amazon.com/GetResource/Console.html?region=eu-west-1&pageLoadStartTime=1379427139127&locale=en_US",
56
+ :user_agent => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36")
35
57
  }
36
58
 
37
- it { should parse_line(sample_copy, 'a COPY line').and_capture(
38
- :bucket_owner => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
39
- :bucket => 'testbucket',
40
- :timestamp => 20100303230459,
41
- :remote_ip => '174.119.31.76',
42
- :key => 'files/image.png',
43
- :operation => 'REST.COPY.OBJECT',
44
- :requester => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
45
- :request_id => 'ACCC34B843C87BC9',
46
- :request_uri => 'PUT /files/image.png HTTP/1.1',
47
- :error_code => nil,
48
- :http_status => 200,
49
- :total_time => 0.365,
50
- :turnaround_time => 0.319,
51
- :bytes_sent => 234,
52
- :object_size => 65957,
53
- :user_agent => '',
54
- :referer => nil)
59
+ it {should parse_line(sample_copy, 'a COPY line').and_capture(
60
+ :bucket_owner => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
61
+ :bucket => 'testbucket',
62
+ :timestamp => 20100303230459,
63
+ :remote_ip => '174.119.31.76',
64
+ :key => 'files/image.png',
65
+ :operation => 'REST.COPY.OBJECT',
66
+ :requester => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
67
+ :request_id => 'ACCC34B843C87BC9',
68
+ :request_uri => 'PUT /files/image.png HTTP/1.1',
69
+ :error_code => nil,
70
+ :http_status => 200,
71
+ :total_time => 0.365,
72
+ :turnaround_time => 0.319,
73
+ :bytes_sent => 234,
74
+ :object_size => 65957,
75
+ :user_agent => '',
76
+ :referer => nil)
55
77
  }
56
78
 
57
79
  it { should_not parse_line('nonsense', 'a nonsense line') }
58
80
  end
59
-
81
+
60
82
  describe '#parse_io' do
61
83
  let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
62
84
  let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
63
-
85
+
64
86
  it "should parse requests correctly and not generate warnings" do
65
87
  log_parser.should_receive(:handle_request).twice
66
88
  log_parser.should_not_receive(:warn)
@@ -26,18 +26,18 @@ describe RequestLogAnalyzer::FileFormat::Apache do
26
26
  line_definition.captures?(:duration).should be_true
27
27
  end
28
28
  end
29
-
29
+
30
30
  describe '.access_line_definition' do
31
31
  it "should parse values in microseconds when no argument is given to %D" do
32
32
  format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
33
33
  format.should parse_line('12345').and_capture(:duration => 0.012345)
34
34
  end
35
-
35
+
36
36
  it "should parse values in microseconds when micro is given as argument to %D" do
37
37
  format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
38
38
  format.should parse_line('12345').and_capture(:duration => 0.012345)
39
39
  end
40
-
40
+
41
41
  it "should parse values in microseconds when micro is given as argument to %D" do
42
42
  format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
43
43
  format.should parse_line('12345').and_capture(:duration => 12.345)
@@ -53,36 +53,50 @@ describe RequestLogAnalyzer::FileFormat::Apache do
53
53
  it { should have(8).report_trackers }
54
54
  end
55
55
 
56
+ context '"vhost_combined" access log parsing' do
57
+ subject { RequestLogAnalyzer::FileFormat.load(:apache, :vhost_combined) }
58
+ describe '#parse_line' do
59
+ let(:sample1) { '72.204.80.86 - psi-equipment.od1.vtiger.com [19/Oct/2013:15:41:54 +0000] "GET /vtiger6/layouts/vlayout/modules/Vtiger/resources/Vtiger.js?&v=5.10.38 HTTP/1.1" 304 - "https://psi-equipment.od1.vtiger.com/vtiger6/index.php?module=PrintTemplates&view=Detail&record=29" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36" 1/683'}
60
+
61
+ it { should parse_line(sample1, 'a sample line').and_capture(
62
+ :remote_host => '72.204.80.86', :remote_logname => nil, :user => nil,
63
+ :vhost => 'psi-equipment.od1.vtiger.com', :duration => 1.0,
64
+ :timestamp => 20131019154154, :http_status => 304, :http_method => 'GET',
65
+ :http_version => '1.1', :bytes_sent => 0)
66
+ }
67
+ end
68
+ end
69
+
56
70
  context '"Common" access log parsing' do
57
- subject { RequestLogAnalyzer::FileFormat.load(:apache, :common) }
58
-
71
+ subject { RequestLogAnalyzer::FileFormat.load(:apache, :common) }
72
+
59
73
  it { should be_well_formed }
60
74
  it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent) }
61
75
  it { should have(6).report_trackers }
62
-
76
+
63
77
  describe '#parse_line' do
64
-
78
+
65
79
  let(:sample1) { '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223' }
66
80
  let(:sample2) { '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414' }
67
-
81
+
68
82
  it { should parse_line(sample1, 'a sample line').and_capture(
69
83
  :remote_host => '1.129.119.13', :remote_logname => nil, :user => nil,
70
84
  :timestamp => 20090908075409, :http_status => 200, :http_method => 'GET',
71
85
  :http_version => '1.0', :bytes_sent => 8223)
72
86
  }
73
-
87
+
74
88
  it { should parse_line(sample2, 'another sample line').and_capture(
75
89
  :remote_host => '1.82.235.29', :remote_logname => nil, :user => nil,
76
90
  :timestamp => 20090908075405, :http_status => 200, :http_method => 'GET',
77
91
  :http_version => '1.1', :bytes_sent => 23414)
78
92
  }
79
-
93
+
80
94
  it { should_not parse_line('nonsense', 'a nonsense line')}
81
95
  end
82
-
96
+
83
97
  describe '#parse_io' do
84
98
  let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
85
-
99
+
86
100
  it "should parse a log snippet successfully without warnings" do
87
101
  log_parser.should_receive(:handle_request).exactly(10).times
88
102
  log_parser.should_not_receive(:warn)
@@ -92,36 +106,36 @@ describe RequestLogAnalyzer::FileFormat::Apache do
92
106
  end
93
107
 
94
108
  context '"Combined" access log parsing' do
95
- subject { RequestLogAnalyzer::FileFormat.load(:apache, :combined) }
96
-
109
+ subject { RequestLogAnalyzer::FileFormat.load(:apache, :combined) }
110
+
97
111
  it { should be_well_formed }
98
112
  it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent, :referer, :user_agent) }
99
113
  it { should have(8).report_trackers }
100
-
114
+
101
115
  describe '#parse_line' do
102
116
  let(:sample1) { '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"' }
103
117
  let(:sample2) { '0:0:0:0:0:0:0:1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"' }
104
-
118
+
105
119
  it { should parse_line(sample1, 'with IPv4 address').and_capture(
106
120
  :remote_host => '69.41.0.45', :remote_logname => nil, :user => nil,
107
121
  :timestamp => 20090902120240, :http_status => 404, :http_method => 'GET',
108
122
  :http_version => '1.1', :bytes_sent => 209, :referer => nil,
109
123
  :user_agent => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
110
124
  }
111
-
125
+
112
126
  it { should parse_line(sample2, 'with IPv6 address').and_capture(
113
127
  :remote_host => '0:0:0:0:0:0:0:1', :remote_logname => nil, :user => nil,
114
128
  :timestamp => 20090902050833, :http_status => 200, :http_method => 'GET',
115
129
  :http_version => '1.1', :bytes_sent => 30, :referer => nil,
116
130
  :user_agent => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
117
131
  }
118
-
132
+
119
133
  it { should_not parse_line('nonsense', 'a nonsense line')}
120
134
  end
121
-
135
+
122
136
  describe '#parse_io' do
123
137
  let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
124
-
138
+
125
139
  it "should parse a log snippet successfully without warnings" do
126
140
  log_parser.should_receive(:handle_request).exactly(5).times
127
141
  log_parser.should_not_receive(:warn)
@@ -31,7 +31,7 @@ describe RequestLogAnalyzer::LineDefinition do
31
31
  end
32
32
 
33
33
  describe '#convert_captured_values' do
34
- let(:request) { mock('request', :convert_value => 'foo') }
34
+ let(:request) { double('request', :convert_value => 'foo') }
35
35
 
36
36
  it "should call convert_value for every captured value" do
37
37
  request.should_receive(:convert_value).twice
@@ -50,8 +50,8 @@ describe RequestLogAnalyzer::LineDefinition do
50
50
  }
51
51
 
52
52
  before do
53
- request.stub!(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
54
- request.stub!(:convert_value).with('baz', anything).and_return('foo')
53
+ request.stub(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
54
+ request.stub(:convert_value).with('baz', anything).and_return('foo')
55
55
  end
56
56
 
57
57
  it "should call Request#convert_value for the initial hash and the value in the hash" do
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: request-log-analyzer
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.12.9
4
+ version: 1.12.10
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-05-21 00:00:00.000000000 Z
13
+ date: 2013-10-20 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: rake
@@ -35,7 +35,7 @@ dependencies:
35
35
  requirements:
36
36
  - - ~>
37
37
  - !ruby/object:Gem::Version
38
- version: '2.13'
38
+ version: '2.14'
39
39
  type: :development
40
40
  prerelease: false
41
41
  version_requirements: !ruby/object:Gem::Requirement
@@ -43,7 +43,7 @@ dependencies:
43
43
  requirements:
44
44
  - - ~>
45
45
  - !ruby/object:Gem::Version
46
- version: '2.13'
46
+ version: '2.14'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: activerecord
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -131,6 +131,9 @@ files:
131
131
  - .travis.yml
132
132
  - DESIGN.rdoc
133
133
  - Gemfile
134
+ - Gemfile.activerecord2
135
+ - Gemfile.activerecord3
136
+ - Gemfile.activerecord4
134
137
  - LICENSE
135
138
  - README.rdoc
136
139
  - Rakefile
@@ -276,7 +279,8 @@ files:
276
279
  - spec/unit/tracker/traffic_tracker_spec.rb
277
280
  - tasks/request_log_analyzer.rake
278
281
  homepage: http://railsdoctors.com
279
- licenses: []
282
+ licenses:
283
+ - MIT
280
284
  post_install_message:
281
285
  rdoc_options:
282
286
  - --title
@@ -295,7 +299,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
295
299
  version: '0'
296
300
  segments:
297
301
  - 0
298
- hash: 4176554289530285347
302
+ hash: -262165736928167385
299
303
  required_rubygems_version: !ruby/object:Gem::Requirement
300
304
  none: false
301
305
  requirements:
@@ -304,7 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
304
308
  version: '0'
305
309
  segments:
306
310
  - 0
307
- hash: 4176554289530285347
311
+ hash: -262165736928167385
308
312
  requirements:
309
313
  - To use the database inserter, ActiveRecord and an appropriate database adapter are
310
314
  required.