wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

data/spec/filter_spec.rb

@@ -0,0 +1,137 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
+
+describe RequestLogAnalyzer::Filter::Timespan, 'both before and after'  do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
+    @filter.prepare
+  end
+  
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+  
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only before'  do
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))    
+    @filter.prepare
+  end
+  
+  it "should accept a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
+  end
+  
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end   
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only after'  do  
+  include RequestLogAnalyzerSpecHelper 
+  
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
+    @filter.prepare
+  end
+  
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+  
+  it "should accept a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
+    @filter.prepare
+  end
+  
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should be_nil
+  end
+  
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should_not be_nil
+  end 
+    
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
+    @filter.prepare
+  end
+  
+  it "should accept a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should_not be_nil
+  end
+  
+  it "should accept a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should_not be_nil
+  end
+
+  it "should reject a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should be_nil
+  end 
+    
+  it "should reject a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
+    @filter.prepare
+  end
+  
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'a working test')).should_not be_nil
+  end
+  
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
+  end  
+end

data/spec/log_processor_spec.rb

@@ -0,0 +1,57 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require 'request_log_analyzer/log_processor'
+
+describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
+
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
+    @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
+  end
+  
+  it "should keep a junk line if :keep_junk_lines is true" do
+    @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
+  end
+  
+  it "should remove a junk line" do
+    @log_anonymizer.anonymize_line("junk line\n").should be_empty
+  end
+
+  it "should keep a teaser line intact" do
+    @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
+  end
+  
+  it "should discard a teaser line if discard_teaser_line is true" do
+    @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
+  end
+  
+  it "should keep a matching line intact if no anonymizing is declared" do
+    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
+  end  
+
+  it "should anonymize values completely if requested" do
+    @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
+  end  
+  
+  it "should anonymize values slightly if requested" do
+    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
+  end
+end
+
+describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
+
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
+  end
+  
+  it "should remove a junk line" do
+    @log_stripper.strip_line("junk line\n").should be_empty
+  end
+
+  it "should keep a teaser line intact" do
+    @log_stripper.strip_line("processing 1234\n").should be_empty
+  end
+end

data/tasks/rspec.rake

@@ -0,0 +1,6 @@
+require 'spec/rake/spectask'
+ 
+desc "Run all specs in spec directory (excluding plugin specs)"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-01 00:00:00 -07:00
+date: 2009-01-12 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -18,57 +18,59 @@ description: Rails log analyzer's purpose is to find what actions are best candi
 email: willem@vanbergen.org
 executables: 
 - request-log-analyzer
-- request-log-database
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
+- DESIGN
+- HACKING
 - LICENSE
 - README.textile
 - Rakefile
-- TODO
 - bin
 - bin/request-log-analyzer
-- bin/request-log-database
 - lib
-- lib/base
-- lib/base/log_parser.rb
-- lib/base/record_inserter.rb
-- lib/base/summarizer.rb
-- lib/bashcolorizer.rb
-- lib/command_line
-- lib/command_line/arguments.rb
-- lib/command_line/exceptions.rb
-- lib/command_line/flag.rb
-- lib/merb_analyzer
-- lib/merb_analyzer/log_parser.rb
-- lib/merb_analyzer/summarizer.rb
-- lib/rails_analyzer
-- lib/rails_analyzer/log_parser.rb
-- lib/rails_analyzer/record_inserter.rb
-- lib/rails_analyzer/summarizer.rb
-- lib/rails_analyzer/virtual_mongrel.rb
+- lib/cli
+- lib/cli/bashcolorizer.rb
+- lib/cli/command_line_arguments.rb
+- lib/cli/progressbar.rb
+- lib/request_log_analyzer
 - lib/request_log_analyzer.rb
-- lib/ruby-progressbar
-- lib/ruby-progressbar/progressbar.en.rd
-- lib/ruby-progressbar/progressbar.ja.rd
-- lib/ruby-progressbar/progressbar.rb
-- output
-- output/blockers.rb
-- output/errors.rb
-- output/hourly_spread.rb
-- output/mean_db_time.rb
-- output/mean_rendering_time.rb
-- output/mean_time.rb
-- output/most_requested.rb
-- output/timespan.rb
-- output/total_db_time.rb
-- output/total_time.rb
-- output/usage.rb
+- lib/request_log_analyzer/aggregator
+- lib/request_log_analyzer/aggregator/base.rb
+- lib/request_log_analyzer/aggregator/database.rb
+- lib/request_log_analyzer/aggregator/echo.rb
+- lib/request_log_analyzer/aggregator/summarizer.rb
+- lib/request_log_analyzer/controller.rb
+- lib/request_log_analyzer/file_format
+- lib/request_log_analyzer/file_format.rb
+- lib/request_log_analyzer/file_format/merb.rb
+- lib/request_log_analyzer/file_format/rails.rb
+- lib/request_log_analyzer/filter
+- lib/request_log_analyzer/filter/base.rb
+- lib/request_log_analyzer/filter/field.rb
+- lib/request_log_analyzer/filter/timespan.rb
+- lib/request_log_analyzer/line_definition.rb
+- lib/request_log_analyzer/log_parser.rb
+- lib/request_log_analyzer/log_processor.rb
+- lib/request_log_analyzer/request.rb
+- lib/request_log_analyzer/source
+- lib/request_log_analyzer/source/base.rb
+- lib/request_log_analyzer/source/log_file.rb
+- lib/request_log_analyzer/tracker
+- lib/request_log_analyzer/tracker/base.rb
+- lib/request_log_analyzer/tracker/category.rb
+- lib/request_log_analyzer/tracker/duration.rb
+- lib/request_log_analyzer/tracker/hourly_spread.rb
+- lib/request_log_analyzer/tracker/timespan.rb
 - spec
 - spec/controller_spec.rb
 - spec/database_inserter_spec.rb
+- spec/file_format_spec.rb
+- spec/file_formats
+- spec/file_formats/spec_format.rb
+- spec/filter_spec.rb
 - spec/fixtures
 - spec/fixtures/merb.log
 - spec/fixtures/multiple_files_1.log
@@ -83,6 +85,7 @@ files:
 - spec/fixtures/test_order.log
 - spec/line_definition_spec.rb
 - spec/log_parser_spec.rb
+- spec/log_processor_spec.rb
 - spec/merb_format_spec.rb
 - spec/rails_format_spec.rb
 - spec/request_spec.rb
@@ -91,19 +94,7 @@ files:
 - tasks
 - tasks/github-gem.rake
 - tasks/request_log_analyzer.rake
-- tasks/test.rake
-- test
-- test/base_summarizer_test.rb
-- test/log_fragments
-- test/log_fragments/fragment_1.log
-- test/log_fragments/fragment_2.log
-- test/log_fragments/fragment_3.log
-- test/log_fragments/fragment_4.log
-- test/log_fragments/fragment_5.log
-- test/log_fragments/merb_1.log
-- test/merb_log_parser_test.rb
-- test/rails_log_parser_test.rb
-- test/record_inserter_test.rb
+- tasks/rspec.rake
 has_rdoc: false
 homepage: http://github.com/wvanbergen/request-log-analyzer/wikis
 post_install_message: 
@@ -131,7 +122,14 @@ signing_key:
 specification_version: 2
 summary: A command line tool to analyze Rails logs
 test_files: 
-- test/base_summarizer_test.rb
-- test/merb_log_parser_test.rb
-- test/rails_log_parser_test.rb
-- test/record_inserter_test.rb
+- spec/controller_spec.rb
+- spec/database_inserter_spec.rb
+- spec/file_format_spec.rb
+- spec/filter_spec.rb
+- spec/line_definition_spec.rb
+- spec/log_parser_spec.rb
+- spec/log_processor_spec.rb
+- spec/merb_format_spec.rb
+- spec/rails_format_spec.rb
+- spec/request_spec.rb
+- spec/summarizer_spec.rb

data/TODO

@@ -1,58 +0,0 @@
-TODO items for Rails-log-analyzer
-=================================
-Contact willem AT vanbergen DOT org if you want to help out with the development.
-
-General:
-- Add more tests / specs
-
-Parameters:
-- Add commandline parameters
-  --version, -v     -> This should support github - last version checking
-  --help, -h        -> Show usage
-
-Datamining:
-- Add query functionality for the resulting database file (interactive reports?)
-- Link request processing line to request completed line (VirtualMongrel?)
-- Fix the database inserter and make it more robust for future changes
-
-Rails integration:
-- Optionally use local or specific routes.rb file to parse URLs
-
-Other:
-- World domination
-
-
-
-
-Datamining should look something like this:
-
-> request-log-analyzer myapp.log --interactive
-Request log analyzer builds a new database.
-Columns come from the log_parser as the LOG_LINES store all the keys that they can detect.
-Also add some extra columns like hashed_request_url etc.
-
-Request log analyzer then parses the logfile for its individual requests using something like the 
-virtual mongrel (we need a new name for this, database_summarizer agregator? sheepdog?) combined with our
-default log parser.
-
-When this is done the user enters an interactive mode (like irb).
-> Filters: None
-> Total requests in database: 53232
-> $
-
-The user can add filters like this:
-> $ FILTER SQL ["date > ?", Date.today-1]
-
-The user will then see this:
-> Filters:
->  1. ["date > ?", Date.today-1]
-> Total requests: 2120
-> $
-
-At any point the user can destroy filters, show the raw requests or show reports
-> $ REPORT ALL
-
-The request remaining after the filter chain will then be processed through the summarizer and then trough the
-output templates, generating reports specificly for the selected dataset.
-Partials should also be possible
-> $ REPORT TIMESPAN

data/bin/request-log-database

@@ -1,81 +0,0 @@
-#!/usr/bin/ruby
-
-require File.dirname(__FILE__) + '/../lib/command_line/arguments'
-require File.dirname(__FILE__) + '/../lib/base/log_parser'
-require File.dirname(__FILE__) + '/../lib/base/record_inserter'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
-require File.dirname(__FILE__) + '/../lib/bashcolorizer'
-require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
-
-
-puts "Rails log parser, by Willem van Bergen and Bart ten Brinke\n\n"
-
-begin
-  
-  $arguments = CommandLine::Arguments.parse do |command_line|
-    command_line.switch(:guess_database_time, :g)
-    command_line.switch(:reset_database, :r)
-    command_line.flag(:database, :alias => :d, :required => false)
-    command_line.required_files = 1
-  end
-  
-rescue CommandLine::Error => e
-  puts "ARGUMENT ERROR: " + e.message
-  puts
-  puts "Usage: ruby parsetodb.rb [LOGFILES*] <OPTIONS>"
-  puts
-  puts "Options:"
-  puts "  --database, -t:             The database file to use"
-  puts "  --reset-database, -r:       Resets the database before inserting new records"
-  puts "  --guess-database-time, -g:  Guesses the database duration of requests"      
-  puts
-  puts "Examples:"
-  puts "  ./parsetodb.rb development.log"
-  puts "  ./parsetodb.rb mongrel.0.log mongrel.1.log mongrel.2.log -g -d mongrel.db"  
-  puts 
-   
-  exit(0) 
-end
-
-log_files  = $arguments.files
-db_file    = $arguments[:database] || log_files.first + '.db'
-
-if $arguments[:reset_database] && File.exist?(db_file) 
-  File.delete(db_file) 
-  puts "Database file cleared."
-end
-
-records_inserted = 0 
-inserter = RailsAnalyzer::RecordInserter.insert_batch_into(db_file) do |db|
-  log_files.each do |log_file|
-
-    puts "Processing all log lines from #{log_file}..."
-    parser = RailsAnalyzer::LogParser.new(log_file)
-    
-    pbar = ProgressBar.new(green(log_file), File.size(log_file))
-    parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
-    
-    parser.each do |request| 
-      db.insert(request) 
-      records_inserted += 1
-    end
-  end
-  
-  if $arguments[:guess_database_time]
-    puts "Calculating database times..."
-    db.calculate_db_durations! 
-  end
-end
-
-started   = inserter.count(:started)
-completed = inserter.count(:completed)
-failed    = inserter.count(:failed)
-
-puts 
-puts "Inserted #{records_inserted} records from #{log_files.length} files."
-puts "Parse warnings: #{inserter.warning_count}. Check the parse_warnings table in the database for details."
-puts
-puts "Requests started: #{started}"
-puts "Requests completed: #{completed}"
-puts "Requests failed: #{failed}"

data/lib/base/log_parser.rb

@@ -1,78 +0,0 @@
-module Base
-  # Parse a log file
-  class LogParser 
-
-    LOG_LINES = {}
-  
-    # LogParser initializer
-    # <tt>file</tt> The fileobject this LogParser wil operate on.
-    def initialize(file, options = {})
-      @file_name = file
-      @options = options
-      @file_size = File.size(@file_name)
-      
-      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
-    end
-  
-    def progress(&block)
-      @progress_handler = block
-    end
-
-    # Output a warning
-    # <tt>message</tt> The warning message (object)
-    def warn(message)
-      puts " -> " + message.to_s
-    end  
-
-    def convert_value(value, type) 
-      return case type
-      when :string;    value.to_s
-      when :int;       value.to_i
-      when :sec;       value.to_f
-      when :msec;      value.to_f / 1000
-      when :timestamp; value.to_s # TODO: fix me?
-      else
-        warn("Unkwn type encountered: #{type}")
-        value
-      end
-    end
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def each(*line_types, &block)
-      log_lines_hash = self.class::LOG_LINES
-
-      
-      # parse everything by default 
-      line_types = log_lines_hash.keys if line_types.empty?
-
-      File.open(@file_name) do |file|
-      
-        file.each_line do |line|
-        
-          @progress_handler.call(file.pos, @file_size) if @progress_handler
-          
-          line_types.each do |line_type|
-            if log_lines_hash[line_type][:teaser] =~ line
-              if log_lines_hash[line_type][:regexp] =~ line
-                
-                captures = $~.captures
-                request = { :type => line_type, :line => file.lineno }
-                log_lines_hash[line_type][:params].each_with_index do |paramhash, index|              
-                  paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
-                end
-                
-                yield(request) if block_given?
-              else
-                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
-              end
-            end            
-          end
-        end
-        @progress_handler.call(:finished, @file_size) if @progress_handler
-      end      
-    end
-  end
-end