request-log-analyzer 1.0.2 → 1.6.3

data/spec/unit/file_format/line_definition_spec.rb

@@ -0,0 +1,75 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::LineDefinition do
+
+  before(:each) do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+      :teaser   => /Testing /,
+      :regexp   => /Testing (\w+), tries\: (\d+)/,
+      :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
+    })
+  end
+
+  describe '#matches' do
+
+    it "should return false on an unmatching line" do
+      @line_definition.matches("nonmatching").should be_false
+    end
+
+    it "should return false when only the teaser matches" do
+      @line_definition.matches("Testing LineDefinition").should be_false
+    end
+
+    it "should parse a line and capture the expected values" do
+      @line_definition.matches("Testing LineDefinition, tries: 123").should == {:line_definition => @line_definition, :captures => ['LineDefinition', '123'] }
+    end
+
+    it "should know which names it can capture" do
+      @line_definition.captures?(:what).should be_true
+      @line_definition.captures?(:tries).should be_true
+      @line_definition.captures?(:bogus).should be_false
+    end
+    
+  end
+
+  describe '#convert_captured_values' do
+
+    before(:each) do
+      @request = mock('request')
+      @request.stub!(:convert_value).and_return('foo')
+    end
+
+    it "should call convert_value for every captured value" do
+      @request.should_receive(:convert_value).twice
+      @line_definition.convert_captured_values(['test', '123'], @request)
+    end
+
+    it "should set the converted values" do
+      @line_definition.convert_captured_values(['test', '123'], @request).should == {:what => 'foo', :tries => 'foo'}
+    end
+
+    context 'when using :provides option' do
+      before(:each) do
+        @ld = RequestLogAnalyzer::LineDefinition.new(:test, :regexp   => /Hash\: (\{.+\})/,
+            :captures => [{ :name => :hash, :type => :hash, :provides => {:bar => :string}}])
+
+        @request = mock('request')
+
+        @request.stub!(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
+        @request.stub!(:convert_value).with('baz', anything).and_return('foo')
+      end
+
+      it "should call Request#convert_value for the initial hash and the value in the hash" do
+        @request.should_receive(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
+        @request.should_receive(:convert_value).with("baz", anything)
+        @ld.convert_captured_values(["{:bar=>'baz'}"], @request)
+      end
+
+      it "should set the provides fields" do
+        # The captures field must be set and converted as well
+        @ld.convert_captured_values(["{:bar=>'baz'}"], @request)[:bar].should eql('foo')
+      end
+    end
+
+  end
+end

data/spec/unit/file_format/merb_format_spec.rb

@@ -0,0 +1,52 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Merb do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:merb).should be_valid
+  end
+
+  describe '#parse_line' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:merb)
+    end
+    
+    it "should parse a :started line correctly" do
+      line = '~ Started request handling: Fri Aug 29 11:10:23 +0200 2008'
+      @file_format.should parse_line(line).as(:started).and_capture(:timestamp => 20080829111023)
+    end
+
+    it "should parse a prefixed :started line correctly" do
+      line = '~ Aug 31 18:35:24 typekit-web001 merb:  ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009'
+      @file_format.should parse_line(line).as(:started).and_capture(:timestamp => 20090831183525)
+    end
+
+    it "should parse a :params line correctly" do
+      line = '~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"delete", "controller"=>"session"}'
+      @file_format.should parse_line(line).as(:params).and_capture(:controller => 'session', :action => 'delete', :namespace => nil)
+    end
+
+    it "should parse a :completed line correctly" do
+      line = '~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}'
+      @file_format.should parse_line(line).as(:completed).and_capture(:dispatch_time => 0.006117, 
+        :before_filters_time => 0.000712, :action_time => 0.005833, :after_filters_time => 6.1e-05)
+    end
+  end
+
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:merb))
+    end
+
+    it "should parse a stream and find valid Merb requests" do
+      @log_parser.parse_file(log_fixture(:merb)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Merb::Request)
+      end
+    end
+  
+    it "should find 11 completed requests" do
+      @log_parser.should_receive(:handle_request).exactly(11).times
+      @log_parser.parse_file(log_fixture(:merb))
+    end
+  end
+end

data/spec/unit/file_format/mysql_format_spec.rb

@@ -0,0 +1,154 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Mysql do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:mysql).should be_valid
+  end
+
+  describe '#parse_line' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:mysql)
+    end
+
+    it "should parse a :time line correctly" do
+      line = '# Time: 091112  8:13:56'
+      @file_format.should parse_line(line).as(:time).and_capture(:timestamp => 20091112081356)
+    end
+
+    it "should parse a :user_host line correctly with IP present" do
+      line = '# User@Host: admin[admin] @ db1 [10.0.0.1]'
+      @file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => 'db1', :ip => '10.0.0.1')
+    end
+
+    it "should parse a :user_host line correctly without a host" do
+      line = '# User@Host: admin[admin] @  [10.0.0.1]'
+      @file_format.should parse_line(line).as(:user_host).and_capture(:user => "admin", :host => '', :ip => '10.0.0.1')
+    end
+
+    it "should parse a :user_host line correctly with IP absent" do
+      line = '# User@Host: root[root] @ localhost []'
+      @file_format.should parse_line(line).as(:user_host).and_capture(:user => "root", :host => 'localhost', :ip => "")
+    end
+
+    it "should parse a :query_statistics line" do
+      line = '# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307'
+      @file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
+          :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
+    end
+
+    it "should parse a :query_statistics line with floating point durations" do
+      line = '# Query_time: 10.00000  Lock_time: 0.00000  Rows_sent: 1191307  Rows_examined: 1191307'
+      @file_format.should parse_line(line).as(:query_statistics).and_capture(:query_time => 10.0,
+          :lock_time => 0.0, :rows_sent => 1191307, :rows_examined => 1191307)
+    end
+    
+    it "should parse a :query_part line" do
+      line = '                               AND clients.index > 0'
+      @file_format.should parse_line(line).as(:query_part).and_capture(:query_fragment => line)
+    end
+    
+    it "should parse a final :query line" do
+      line = 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`; '
+      @file_format.should parse_line(line).as(:query).and_capture(:query =>
+        'SELECT /*!:int SQL_NO_CACHE */ * FROM events')
+    end
+    
+    it "should parse a :use_database line" do
+      line = 'use db;'
+      @file_format.should parse_line(line).as(:use_database).and_capture(:database => 'db')
+    end
+    
+    it "should not parse a SET timestamp line" do
+      line = 'SET timestamp=1250651725;'
+      @file_format.should_not parse_line(line)
+    end
+
+    it "should not parse a SET insert_id line" do
+      line = 'SET insert_id=1250651725;'
+      @file_format.should_not parse_line(line)
+    end
+
+    it "should not parse a SET timestamp, insert_id line" do
+      line = 'SET timestamp=1250651725, insert_id=45674;'
+      @file_format.should_not parse_line(line)
+    end
+
+
+  end
+
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:mysql))
+    end
+
+    it "should parse a single line query entry correctly" do
+      fixture = <<EOS
+# Time: 091112 18:13:56
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+EOS
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
+        request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
+      end
+    end
+
+    it "should parse a multiline query entry correctly" do
+      fixture = <<EOS
+# Time: 091112 18:13:56
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+SELECT * FROM `clients` WHERE (1=1 
+                              AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
+                               AND clients.index > 0
+                              ) AND (clients.deleted_at IS NULL);
+EOS
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql::Request)
+        request[:query].should == "SELECT * FROM clients WHERE (:int=:int AND clients.valid_from < :date AND (clients.valid_to IS NULL or clients.valid_to > :date) AND clients.index > :int ) AND (clients.deleted_at IS NULL)"
+      end
+    end
+
+    it "should parse a request without timestamp correctly" do
+        fixture = <<EOS
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+EOS
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
+      end
+    end
+
+    it "should parse a query with context information correctly" do
+      fixture = <<EOS
+# Time: 091112 18:13:56
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+use database_name;
+SET timestamp=4324342342423, insert_id = 224253443;
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+EOS
+
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Mysql::Request) && request.completed?
+        request[:query].should == 'SELECT /*!:int SQL_NO_CACHE */ * FROM events'
+      end
+
+    end
+
+
+    it "should find 26 completed sloq query entries" do
+      @log_parser.should_receive(:handle_request).exactly(26).times
+      @log_parser.parse_file(log_fixture(:mysql_slow_query))
+    end
+  end
+end

data/spec/unit/file_format/postgresql_format_spec.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Postgresql do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:Postgresql).should be_valid
+  end
+
+  describe '#parse_line' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
+    end
+
+    it "should parse a :query line correctly" do
+      line = '2004-05-07 11:58:36 LOG:  query: SELECT plugin_id, plugin_name FROM plugins'
+      @file_format.should parse_line(line).as(:query).and_capture(:timestamp => 20040507115836, :query_fragment => 'SELECT plugin_id, plugin_name FROM plugins')
+    end
+    
+    it "should parse a :query_fragment line correctly" do
+      line = '    groups.type_id,users.user_name,users.realname,'
+      @file_format.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => "groups.type_id,users.user_name,users.realname,")
+    end
+
+    it "should parse a :duration line correctly" do
+      line = '2004-05-07 11:58:36 LOG:  duration: 0.002612 sec'
+      @file_format.should parse_line(line).as(:duration).and_capture(:query_time => 0.002612)
+    end    
+  end
+
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
+    end
+
+    it "should parse a multiline query entry correctly" do
+      fixture = <<EOS
+2004-05-07 11:58:22 LOG:  query: SELECT groups.group_name,groups.unix_group_name,
+		groups.type_id,users.user_name,users.realname,
+		news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details 
+		FROM users,news_bytes,groups 
+		WHERE news_bytes.group_id='98' AND news_bytes.is_approved <> '4' 
+		AND users.user_id=news_bytes.submitted_by 
+		AND news_bytes.group_id=groups.group_id 
+		ORDER BY post_date DESC LIMIT 10 OFFSET 0
+2004-05-07 11:58:22 LOG:  duration: 0.002680 sec
+EOS
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
+        request[:query].should == "SELECT groups.group_name,groups.unix_group_name, groups.type_id,users.user_name,users.realname, news_bytes.forum_id,news_bytes.summary,news_bytes.post_date,news_bytes.details FROM users,news_bytes,groups WHERE news_bytes.group_id=:string AND news_bytes.is_approved <> :string AND users.user_id=news_bytes.submitted_by AND news_bytes.group_id=groups.group_id ORDER BY post_date DESC LIMIT :int OFFSET :int"
+      end
+    end
+
+    it "should parse a dualline query entry correctly" do
+      fixture = <<EOS
+      2004-05-07 11:58:36 LOG:  query: SELECT type, count FROM project_sums_agg WHERE group_id=59
+      2004-05-07 11:58:36 LOG:  duration: 0.001197 sec
+EOS
+      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Postgresql::Request)
+        request[:query].should == "SELECT type, count FROM project_sums_agg WHERE group_id=:int"
+      end
+    end
+  end
+end
+

data/spec/unit/file_format/rack_format_spec.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Rack do
+
+  context '"Sinatra" access log parsing' do
+
+    before(:each) do
+      @file_format      = RequestLogAnalyzer::FileFormat.load(:rack)
+      @log_parser       = RequestLogAnalyzer::Source::LogParser.new(@file_format)    
+      @sample = '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024'
+    end
+
+    it "should be a valid file format" do
+      @file_format.should be_valid
+    end
+
+    it "should parse access lines and capture all of its fields" do
+      @file_format.should have_line_definition(:access).capturing(:remote_host, :timestamp, :http_method, :path, :http_version, 
+        :http_status, :bytes_sent, :duration)
+    end
+
+    it "should match the sample line" do
+      @file_format.parse_line(@sample).should include(:line_definition, :captures)
+    end
+
+    it "should not match a nonsense line" do
+      @file_format.parse_line('== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel').should be_nil
+    end
+
+    it "should parse the sample fields correctly" do
+      match = @file_format.parse_line(@sample)
+      match.should_not be_nil
+    end
+
+    it "should parse and convert the sample fields correctly" do
+      @log_parser.parse_io(StringIO.new(@sample)) do |request|
+        request[:remote_host].should    == '127.0.0.1'
+        request[:timestamp].should      == 20091123214747
+        request[:http_method].should    == 'GET'
+        request[:path].should           == '/css/stylesheet.css'
+        request[:http_version].should   == '1.1'
+        request[:http_status].should    == 200
+        request[:bytes_sent].should     == 3782
+        request[:duration].should       == 0.0024
+      end
+    end
+
+  end
+
+end

data/spec/unit/file_format/rails3_format_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Rails do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:rails3).should be_valid
+  end
+  
+  describe '#parse_line' do
+    before(:each) { @file_format = RequestLogAnalyzer::FileFormat.load(:rails3) }
+
+    it "should parse :started lines correctly" do
+      line = 'Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18'
+      @file_format.should parse_line(line).as(:started).and_capture(:method => 'GET', 
+            :path => '/queries', :ip => '127.0.0.1', :timestamp => 20100225161518)
+    end
+    
+    it "should parse :processing lines correctly" do
+      line = ' Processing by QueriesController#index as HTML'
+      @file_format.should parse_line(line).as(:processing).and_capture(
+        :controller => 'QueriesController', :action => 'index', :format => 'HTML')
+    end
+    
+    
+    # it "should parse beta :completed lines correctly" do
+    #   line = 'Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200'
+    #   @file_format.should parse_line(line).as(:completed).and_capture(
+    #       :duration => 0.009, :status => 200)
+    # end
+    
+    it "should parse :completed lines correctly" do
+      line = 'Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)'
+      @file_format.should parse_line(line).as(:completed).and_capture(
+          :duration => 0.170, :status => 200)
+    end
+    
+    it "should pase :failure lines correctly" do
+      line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"
+      @file_format.should parse_line(line).as(:failure).and_capture(:line => 3, 
+          :error   => 'ActionView::Template::Error', 
+          :message => "undefined local variable or method `field' for #<Class>", 
+          :file    => '/Users/willem/Code/warehouse/app/views/queries/execute.csv.erb')
+    end
+  end
+  
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails3))
+    end
+    
+    it "should parse a successful request correctly" do
+      log = <<-EOLOG
+        Started GET "/" for 127.0.0.1 at 2010-03-19 06:40:41
+          Processing by QueriesController#index as HTML
+          SQL (16.3ms)  SHOW TABLES
+          Query Load (32.0ms)  SELECT `queries`.* FROM `queries`
+        Rendered queries/index.html.erb within layouts/default (40.9ms)
+        Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)
+      EOLOG
+      
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+    
+    it "should parse an unroutable request correctly" do
+      log = <<-EOLOG
+        Started GET "/404" for 127.0.0.1 at 2010-03-19 06:40:57
+
+        ActionController::RoutingError (No route matches "/404"):
+
+
+        Rendered /Users/rails/actionpack/lib/action_dispatch/middleware/templates/rescues/routing_error.erb within rescues/layout (1.0ms)
+      
+      EOLOG
+
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+    
+    it "should parse a failing request correctly" do
+      log = <<-EOLOG
+        Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at 2010-03-01 18:44:33
+          Processing by QueriesController#execute as CSV
+          Parameters: {"commit"=>"Run query", "authenticity_token"=>"pz9WcxkcrlG/43eg6BgSAnJL7yIsaffuHbYxPHUsUzQ=", "id"=>"397638749"}
+
+        ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/application/app/views/queries/execute.csv.erb:
+        1: <%=raw @result.fields.map { |f| f.humanize.to_json }.join(',') %>
+        2: <% @result.each do |record| %>
+        3:   <%=raw @result.fields.map { |f| record[field].to_s }.join(",") %>
+        4: <% end %>
+
+            app/views/queries/execute.csv.erb:3:in `_render_template__652100315_2182241460_0'
+            app/views/queries/execute.csv.erb:3:in `map'
+            app/views/queries/execute.csv.erb:3:in `_render_template__652100315_2182241460_0'
+            app/views/queries/execute.csv.erb:2:in `_render_template__652100315_2182241460_0'
+            app/controllers/queries_controller.rb:34:in `execute'
+
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/_trace.erb (1.0ms)
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/_request_and_response.erb (9.7ms)
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/template_error.erb within /Users/willem/.rvm/gems/ruby-1.8.7-p248/gems/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/layout.erb (20.4ms)
+
+      EOLOG
+
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+  end
+end