request-log-analyzer 1.9.9 → 1.10.0

data/spec/spec_helper.rb

@@ -1,8 +1,9 @@
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-
 require 'rubygems'
+require 'bundler/setup'
+
 require 'rspec'
 require 'request_log_analyzer'
+require 'stringio'
 
 module RequestLogAnalyzer::RSpec
 end

data/spec/unit/file_format/apache_format_spec.rb

@@ -28,6 +28,23 @@ describe RequestLogAnalyzer::FileFormat::Apache do
       @line_definition.captures?(:duration).should be_true
     end
   end
+  
+  describe '.access_line_definition' do
+    it "should parse values in microseconds when no argument is given to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
+      format.should parse_line('12345').and_capture(:duration => 0.012345)
+    end
+    
+    it "should parse values in microseconds when micro is given as argument to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
+      format.should parse_line('12345').and_capture(:duration => 0.012345)
+    end
+    
+    it "should parse values in microseconds when micro is given as argument to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
+      format.should parse_line('12345').and_capture(:duration => 12.345)
+    end
+  end
 
   describe '.create' do
 

data/spec/unit/file_format/haproxy_format_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Haproxy do
+
+  before(:each) do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:haproxy)
+    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+    @sample1 = 'Feb  6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1"'
+    @sample2 = 'haproxy[18113]: 127.0.0.1:34549 [15/Oct/2003:15:19:06.103] px-http px-http/<NOSRV> -1/-1/-1/-1/+50001 408 +2750 - - cR-- 2/2/2/0/+2 0/0 ""'
+  end
+
+  it "should be a valid file format" do
+    @file_format.should be_valid
+  end
+
+  it "should parse access lines and capture all of its fields" do
+    @file_format.should have_line_definition(:haproxy).capturing(:client_ip, :accept_date, :frontend_name, :backend_name, :server_name, :tq, :tw, :tc, :tr, :tt, :status_code, :bytes_read, :captured_request_cookie, :captured_response_cookie, :termination_event_code, :terminated_session_state, :clientside_persistence_cookie, :serverside_persistence_cookie, :actconn, :feconn, :beconn, :srv_conn, :retries, :srv_queue, :backend_queue, :captured_request_headers, :captured_response_headers, :http_request)
+  end
+
+  it "should match the sample line" do
+    @file_format.parse_line(@sample1).should include(:line_definition, :captures)
+  end
+
+  it "should not match a nonsense line" do
+    @file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
+  end
+
+  it "should parse and convert the sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample1)) do |request|
+      request[:client_ip].should                      == '10.0.1.2'
+      request[:accept_date].should                    == 20090206121414
+      request[:frontend_name].should                  == 'http-in'
+      request[:backend_name].should                   == 'static'
+      request[:server_name].should                    == 'srv1'
+      request[:tq].should                             == 0.010
+      request[:tw].should                             == 0.000
+      request[:tc].should                             == 0.030
+      request[:tr].should                             == 0.069
+      request[:tt].should                             == 0.109
+      request[:status_code].should                    == 200
+      request[:bytes_read].should                     == 2750
+      request[:captured_request_cookie].should        == nil
+      request[:captured_response_cookie].should       == nil
+      request[:termination_event_code].should         == nil
+      request[:terminated_session_state].should       == nil
+      request[:clientside_persistence_cookie].should  == nil
+      request[:serverside_persistence_cookie].should  == nil
+      request[:actconn].should                        == 1
+      request[:feconn].should                         == 1
+      request[:beconn].should                         == 1
+      request[:srv_conn].should                       == 1
+      request[:retries].should                        == 0
+      request[:srv_queue].should                      == 0
+      request[:backend_queue].should                  == 0
+      request[:captured_request_headers].should       == '{1wt.eu}'
+      request[:captured_response_headers].should      == nil
+      request[:http_request].should                   == 'GET /index.html HTTP/1.1'
+    end
+  end
+
+  it "should parse and convert edge case sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample2)) do |request|
+      request[:accept_date].should                    == 20031015151906
+      request[:server_name].should                    == '<NOSRV>'
+      request[:tq].should                             == nil
+      request[:tw].should                             == nil
+      request[:tc].should                             == nil
+      request[:tr].should                             == nil
+      request[:tt].should                             == 50.001
+      request[:bytes_read].should                     == 2750
+      request[:captured_request_cookie].should        == nil
+      request[:captured_response_cookie].should       == nil
+      request[:termination_event_code].should         == 'c'
+      request[:terminated_session_state].should       == 'R'
+      request[:clientside_persistence_cookie].should  == nil
+      request[:serverside_persistence_cookie].should  == nil
+      request[:retries].should                        == 2
+      request[:captured_request_headers].should       == nil
+      request[:captured_response_headers].should      == nil
+      request[:http_request].should                   == nil
+    end
+  end
+
+end

data/spec/unit/file_format/postgresql_format_spec.rb

@@ -2,64 +2,42 @@ require 'spec_helper'
 
 describe RequestLogAnalyzer::FileFormat::Postgresql do
 
-  it "should be a valid file format" do
-    RequestLogAnalyzer::FileFormat.load(:Postgresql).should be_valid
-  end
+  subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
+  let(:log_parser)  { RequestLogAnalyzer::Source::LogParser.new(subject) }
+  
+  it { should be_valid }
 
   describe '#parse_line' do
-    before(:each) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
-    end
-
     it "should parse a :query line correctly" do
-      line = '2004-05-07 11:58:36 LOG:  query: SELECT plugin_id, plugin_name FROM plugins'
-      @file_format.should parse_line(line).as(:query).and_capture(:timestamp => 20040507115836, :query_fragment => 'SELECT plugin_id, plugin_name FROM plugins')
+      line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG:  00000: duration: 0.710 ms  statement: SELECT * FROM "delayed_jobs"'
+      subject.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
     end
     
     it "should parse a :query_fragment line correctly" do
-      line = '    groups.type_id,users.user_name,users.realname,'
-      @file_format.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => "groups.type_id,users.user_name,users.realname,")
+      line = '    ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
+      subject.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
     end
 
-    it "should parse a :duration line correctly" do
-      line = '2004-05-07 11:58:36 LOG:  duration: 0.002612 sec'
-      @file_format.should parse_line(line).as(:duration).and_capture(:query_time => 0.002612)
-    end    
+    it "should parse a :query line correctly" do
+      line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG:  00000: duration: 0.710 ms  statement: SELECT * FROM "delayed_jobs"'
+      subject.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
+    end
   end
 
   describe '#parse_io' do
-    before(:each) do
-      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
-    end
-
     it "should parse a multiline query entry correctly" do
-      fixture = <<EOS
-2004-05-07 11:58:22 LOG:  query: SELECT groups.group_name,groups.unix_group_name,
-		groups.type_id,users.user_name,users.realname,
-		news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details 
-		FROM users,news_bytes,groups 
-		WHERE news_bytes.group_id='98' AND news_bytes.is_approved <> '4' 
-		AND users.user_id=news_bytes.submitted_by 
-		AND news_bytes.group_id=groups.group_id 
-		ORDER BY post_date DESC LIMIT 10 OFFSET 0
-2004-05-07 11:58:22 LOG:  duration: 0.002680 sec
-EOS
-      @log_parser.parse_io(StringIO.new(fixture)) do |request|
-        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
-        request[:query].should == "SELECT groups.group_name,groups.unix_group_name, groups.type_id,users.user_name,users.realname, news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details FROM users,news_bytes,groups WHERE news_bytes.group_id=:string AND news_bytes.is_approved <> :string AND users.user_id=news_bytes.submitted_by AND news_bytes.group_id=groups.group_id ORDER BY post_date DESC LIMIT :int OFFSET :int"
-      end
-    end
-
-    it "should parse a dualline query entry correctly" do
-      fixture = <<EOS
-      2004-05-07 11:58:36 LOG:  query: SELECT type, count FROM project_sums_agg WHERE group_id=59
-      2004-05-07 11:58:36 LOG:  duration: 0.001197 sec
-EOS
-      @log_parser.parse_io(StringIO.new(fixture)) do |request|
-        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
-        request[:query].should == "SELECT type, count FROM project_sums_agg WHERE group_id=:int"
+      fixture = <<-EOS
+        2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG:  00000: duration: 0.195 ms  statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {}
+        	', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id"
+        2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION:  exec_simple_query, postgres.c:1081
+      EOS
+
+      request_counter.should_receive(:hit!).exactly(1).times
+      log_parser.should_not_receive(:warn)
+      log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Postgresql::Request) && request.completed?
+        request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
       end
     end
   end
 end
-

data/spec/unit/file_format/rails3_format_spec.rb

@@ -44,9 +44,9 @@ describe RequestLogAnalyzer::FileFormat::Rails do
     end
     
     it "should parse :completed lines correctly" do
-      line = 'Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)'
+      line = 'Completed 200 OK in 170ms (Views: 78.0ms | ActiveRecord: 48.2ms)'
       @file_format.should parse_line(line).as(:completed).and_capture(
-          :duration => 0.170, :status => 200)
+          :duration => 0.170, :view => 0.078, :db => 0.0482, :status => 200)
     end
     
     it "should parse :completed lines correctly when ActiveRecord is not mentioned" do
@@ -54,6 +54,12 @@ describe RequestLogAnalyzer::FileFormat::Rails do
       @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.364, :status => 200)
     end
     
+    it "should parse :completed lines correctly when other durations are specified" do
+      line = 'Completed 200 OK in 384ms (Views: 222.0ms | ActiveRecord: 121.0ms | Sphinx: 0.0ms)'
+      @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.384, :view => 0.222, 
+          :db => 0.121, :status => 200)
+    end
+    
 
     it "should pase :failure lines correctly" do
       line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"

data/spec/unit/file_format/w3c_format_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::W3c do
+
+  before(:each) do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:w3c)
+    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)  
+    #          date       time     c-ip cs-username s-ip     s-port  cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(User-Agent) cs(Referrer) 
+    @sample = '2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/'
+  end
+
+  it "should be a valid file format" do
+    @file_format.should be_valid
+  end
+
+  it "should parse access lines and capture all of its fields" do
+    @file_format.should have_line_definition(:access).capturing(:timestamp, :remote_ip, :username, :local_ip, :port, :method, :path, :http_status, :bytes_sent, :bytes_received, :duration, :user_agent, :referer)
+  end
+
+  it "should match the sample line" do
+    @file_format.parse_line(@sample).should include(:line_definition, :captures)
+  end
+
+  it "should not match a nonsense line" do
+    @file_format.parse_line('#Software: Microsoft Internet Information Services 6.0').should be_nil
+  end
+
+  it "should parse and convert the sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample)) do |request|
+      request[:timestamp].should      == 20020524201801
+      request[:remote_ip].should      == "172.224.24.114"
+      request[:username].should       == nil
+      request[:local_ip].should       == "206.73.118.24"
+      request[:port].should           == 80
+      request[:method].should         == 'GET'
+      request[:path].should           == '/Default.htm'
+      request[:http_status].should    == 200
+      request[:bytes_sent].should     == 7930
+      request[:bytes_received].should == 248
+      request[:duration].should       == 0.031
+      request[:user_agent].should     == 'Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server)'
+      request[:referer].should        == 'http://64.224.24.114/'
+    end
+  end
+
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 9
-  - 9
-  version: 1.9.9
+  - 10
+  - 0
+  version: 1.10.0
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-17 00:00:00 -05:00
+date: 2011-02-26 00:00:00 -05:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -84,6 +84,7 @@ extra_rdoc_files:
 files: 
 - .gitignore
 - DESIGN.rdoc
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -112,6 +113,7 @@ files:
 - lib/request_log_analyzer/file_format/delayed_job.rb
 - lib/request_log_analyzer/file_format/delayed_job2.rb
 - lib/request_log_analyzer/file_format/delayed_job21.rb
+- lib/request_log_analyzer/file_format/haproxy.rb
 - lib/request_log_analyzer/file_format/merb.rb
 - lib/request_log_analyzer/file_format/mysql.rb
 - lib/request_log_analyzer/file_format/oink.rb
@@ -120,6 +122,7 @@ files:
 - lib/request_log_analyzer/file_format/rails.rb
 - lib/request_log_analyzer/file_format/rails3.rb
 - lib/request_log_analyzer/file_format/rails_development.rb
+- lib/request_log_analyzer/file_format/w3c.rb
 - lib/request_log_analyzer/filter.rb
 - lib/request_log_analyzer/filter/anonymize.rb
 - lib/request_log_analyzer/filter/field.rb
@@ -196,6 +199,7 @@ files:
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/file_format/file_format_api_spec.rb
 - spec/unit/file_format/format_autodetection_spec.rb
+- spec/unit/file_format/haproxy_format_spec.rb
 - spec/unit/file_format/line_definition_spec.rb
 - spec/unit/file_format/merb_format_spec.rb
 - spec/unit/file_format/mysql_format_spec.rb
@@ -204,6 +208,7 @@ files:
 - spec/unit/file_format/rack_format_spec.rb
 - spec/unit/file_format/rails3_format_spec.rb
 - spec/unit/file_format/rails_format_spec.rb
+- spec/unit/file_format/w3c_format_spec.rb
 - spec/unit/filter/anonymize_filter_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/filter/filter_spec.rb
@@ -277,6 +282,7 @@ test_files:
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/file_format/file_format_api_spec.rb
 - spec/unit/file_format/format_autodetection_spec.rb
+- spec/unit/file_format/haproxy_format_spec.rb
 - spec/unit/file_format/line_definition_spec.rb
 - spec/unit/file_format/merb_format_spec.rb
 - spec/unit/file_format/mysql_format_spec.rb
@@ -285,6 +291,7 @@ test_files:
 - spec/unit/file_format/rack_format_spec.rb
 - spec/unit/file_format/rails3_format_spec.rb
 - spec/unit/file_format/rails_format_spec.rb
+- spec/unit/file_format/w3c_format_spec.rb
 - spec/unit/filter/anonymize_filter_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/filter/filter_spec.rb