wvanbergen-request-log-analyzer 0.3.4 → 1.0.0

data/README.textile

@@ -17,16 +17,29 @@ h2. Installation
 h2. Usage
 
 <pre>
-Usage: request-log-analyzer [FILE] [OPTION]
-Analyze the given log FILE with the given OPTION
-Example: request-log-analyzer mongrel.log -z
-
-  --fast, -f:                 Only use completed requests
-  --guess-database-time, -g:  Guesses the database duration of requests if they are not in the log
-  --output, -o:               Comma-separated list of reports to show    
-  --amount, -c:               Displays the top <amount> elements in the reports
-  --colorize, -z:             Fancy bash coloring
-  --install rails, -i rails:  Install Rails task rake log:analyze
+  Usage: request-log-analyzer [LOGFILES*] <OPTIONS>
+
+  Input options:
+    --format <format>, -f:     Uses the specified log file format. Defaults to rails.
+    --after <date>             Only consider requests from <date> or later.
+    --before <date>            Only consider requests before <date>.
+    --select <field> <value>   Only consider requests where <field> matches <value>.
+    --reject <field> <value>   Only consider requests where <field> does not match <value>.
+
+  Output options:
+    --boring, -b               Output reports without ASCII colors.
+    --database <filename>, -d: Creates an SQLite3 database of all the parsed request information.
+    --debug                    Print debug information while parsing.
+
+  Examples:
+    request-log-analyzer development.log
+    request-log-analyzer -z mongrel.0.log mongrel.1.log mongrel.2.log 
+    request-log-analyzer --format merb -d requests.db production.log
+
+  To install rake tasks in your Rails application, 
+  run the following command in your application's root directory:
+
+    request-log-analyzer install rails
 </pre>
 
 

data/TODO

@@ -5,6 +5,11 @@ Contact willem AT vanbergen DOT org if you want to help out with the development
 General:
 - Add more tests / specs
 
+Parameters:
+- Add commandline parameters
+  --version, -v     -> This should support github - last version checking
+  --help, -h        -> Show usage
+
 Datamining:
 - Add query functionality for the resulting database file (interactive reports?)
 - Link request processing line to request completed line (VirtualMongrel?)

data/bin/request-log-analyzer

@@ -1,31 +1,94 @@
 #!/usr/bin/ruby
 require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
-require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
 
-puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
+def terminal_width(default = 81)
+  IO.popen('stty -a') do |pipe|
+    column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+    width = column_line ? $1.to_i : default
+  end
+rescue 
+  default  
+end
 
 # Parse the arguments given via commandline
 begin
   arguments = CommandLine::Arguments.parse do |command_line|
-    command_line.switch(:guess_database_time, :g)
-    command_line.switch(:fast, :f)
-    command_line.switch(:colorize, :z)
-    command_line.switch(:merb, :m)
-    command_line.switch(:install, :i)
-    command_line.flag(:output, :alias => :o)
-    command_line.flag(:amount, :alias => :c)
-    command_line.required_files = 1
+
+    command_line.command(:install) do |install|
+      install.parameters = 1
+    end
+    
+    command_line.command(:strip) do |strip|
+      strip.minimum_parameters = 1
+      strip.option(:format, :alias => :f, :default => 'rails')      
+      strip.option(:output, :alias => :o)
+      strip.switch(:discard_teaser_lines, :t)
+      strip.switch(:keep_junk_lines, :j)      
+    end
+    
+    command_line.command(:anonymize) do |anonymize|
+      anonymize.minimum_parameters = 1
+      anonymize.option(:format, :alias => :f, :default => 'rails')            
+      anonymize.option(:output, :alias => :o)
+      anonymize.switch(:discard_teaser_lines, :t)
+      anonymize.switch(:keep_junk_lines, :j)      
+    end
+    
+    command_line.option(:format, :alias => :f, :default => 'rails')
+    command_line.option(:file, :alias => :e)
+    command_line.switch(:single_lines, :s)
+    command_line.switch(:assume_correct_order)
+    
+    command_line.option(:aggregator, :alias => :a, :multiple => true)
+    command_line.option(:database, :alias => :d)    
+
+    # filtering options
+    command_line.option(:select, :multiple => true, :parameters => 2)
+    command_line.option(:reject, :multiple => true, :parameters => 2)
+    command_line.option(:after)
+    command_line.option(:before)    
+    
+    command_line.switch(:boring, :b)
+    command_line.option(:report_width, :default => terminal_width - 1)
+    
+    command_line.switch(:debug)
+    
+    command_line.minimum_parameters = 1    
   end
   
 rescue CommandLine::Error => e
-  puts "ARGUMENT ERROR: " + e.message
+  puts "ARGUMENT ERROR: " + e.message if e.message
   puts
-  load File.dirname(__FILE__) + "/../output/usage.rb"
+  puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
+  puts
+  puts "Input options:"
+  puts "  --format <format>, -f:     Uses the specified log file format. Defaults to rails."
+  puts "  --after <date>             Only consider requests from <date> or later."
+  puts "  --before <date>            Only consider requests before <date>."  
+  puts "  --select <field> <value>   Only consider requests where <field> matches <value>."        
+  puts "  --reject <field> <value>   Only consider requests where <field> does not match <value>."    
+  puts
+  puts "Output options:"
+  puts "  --boring, -b               Output reports without ASCII colors."
+  puts "  --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
+  puts "  --debug                    Print debug information while parsing."
+  puts "  --file <filename>          Output to file."
+  puts 
+  puts "Examples:"
+  puts "  request-log-analyzer development.log"
+  puts "  request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
+  puts "  request-log-analyzer --format merb -d requests.db production.log"
+  puts
+  puts "To install rake tasks in your Rails application, "
+  puts "run the following command in your application's root directory:"
+  puts
+  puts "  request-log-analyzer install rails"
   exit(0) 
 end
 
-if arguments[:install]
-  if arguments.files.first == 'rails'
+def install_rake_tasks(install_type)
+  if install_type == 'rails'
     require 'ftools'
     if File.directory?('./lib/tasks/')
       File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
@@ -35,14 +98,27 @@ if arguments[:install]
       puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
       puts "Installation aborted."
     end
+  else
+    raise "Cannot perform this install type! (#{install_type})"
   end
-  exit(0)
 end
 
-$colorize = true if arguments[:colorize]
-
-# Run the request_log_analyzer!
-request_log_analyzer = RequestLogAnalyzer.new(arguments)
-request_log_analyzer.analyze_this(arguments.files)
 
+case arguments.command
+when :install
+  install_rake_tasks(arguments.parameters[0])
+when :strip
+  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'  
+  RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
+when :anonymize
+  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'    
+  RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!  
+else 
+  puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke - Version 0.4.0\n\n"  
+  
+  # Run the request_log_analyzer!
+  RequestLogAnalyzer::Controller.build(arguments, terminal_width).run!
 
+  puts 
+  puts "Thanks for using request-log-analyzer"
+end

data/lib/request_log_analyzer.rb

@@ -1,161 +1,14 @@
-require File.dirname(__FILE__) + '/../lib/base/log_parser'
-require File.dirname(__FILE__) + '/../lib/base/summarizer'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/summarizer'
-require File.dirname(__FILE__) + '/../lib/rails_analyzer/virtual_mongrel.rb'
-require File.dirname(__FILE__) + '/../lib/merb_analyzer/log_parser'
-require File.dirname(__FILE__) + '/../lib/merb_analyzer/summarizer'
-require File.dirname(__FILE__) + '/../lib/bashcolorizer'
-require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
-
-# Can calculate request counts, duratations, mean times etc. of all the requests given.
-class RequestLogAnalyzer
-  attr_reader :log_parser_class
-  attr_reader :summerizer
-  attr_reader :arguments
-  attr_reader :line_types
-  attr_reader :amount
-  attr_reader :fast
-
-  # Initializer. Sets global variables
-  # Options
-  # *<tt>:fast</tt> Only look at request initializers. Faster, but not not all outputs are shown.
-  # *<tt>:guess_database_time</tt> Guess database time if it it not in the log (default for Rails produciton).
-  # *<tt>:merb</tt> Use merb summarizer and parser classes instead of rails.
-  # *<tt>:output_reports</tt> Comma separated string of requested output reports
-  # *<tt>:amount</tt> Amount of lines shown for each result table. Defaults to 10.
-  def initialize(options = {})
-    @fast                 = options[:fast] || false
-    @guess_database_time  = options[:guess_database_time] || false
-
-    if options[:merb]
-      @summarizer       = MerbAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
-      @log_parser_class = MerbAnalyzer::LogParser
-    else
-      @summarizer       = RailsAnalyzer::Summarizer.new(:calculate_database => @guess_database_time)
-      @log_parser_class = RailsAnalyzer::LogParser
-    end
-
-    @output_reports = options[:output].split(', ') rescue [:timespan, :most_requested, :total_time, :mean_time, :total_db_time, :mean_db_time, :mean_rendering_time, :blockers, :hourly_spread, :errors] 
-    @amount       = options[:amount] || 10
-
-    @line_types = @log_parser_class::LOG_LINES.keys
-    @line_types = [:completed] if @fast
-  end
- 
-
-  # Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
-  # This is used to aggregate simular requests. 
-  # <tt>request</tt> The request to evaluate.
-  # Returns uniformed url string.
-  # Raises on mailformed request.
-  def request_hasher(request)
-    if request[:url]
-      url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
-      url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
-
-      url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
-      url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
-      url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
-
-      return url
-    elsif request[:controller] && request[:action]
-      return "#{request[:controller]}##{request[:action]}"
-    else
-      raise 'Cannot hash this request! ' + request.inspect
-    end
-  end
-
-  # Print results using a ASCII table.
-  # <tt>summarizer</tt> The summarizer containg information to draw the table.
-  # <tt>field</tt> The field containing the data to be printed
-  # <tt>amount</tt> The length of the table (defaults to 20)
-  def print_table(summarizer, field, amount = @amount)
-    summarizer.sort_actions_by(field).reverse[0, amount.to_i].each do |a|
-      # As we show count by default, show totaltime if we sort by count
-      field = :total_time if field == :count
-
-      puts "%-50s: %10.03fs [#{green("%d requests")}]" % [a[0], a[1][field], a[1][:count]]
-    end
-  end
-
-  # Execute the analyze
-  def analyze_this(files = [])
-    # Walk through al the files given via the arguments.
-    files.each do |log_file|
-      puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
-
-      parser = @log_parser_class.new(log_file)
-
-      # add progress bar 
-      unless @fast
-        pbar = ProgressBar.new(green(log_file), File.size(log_file))
-        parser.progress { |pos, total| (pos == :finished) ? pbar.finish : pbar.set(pos) }
-      end
-
-      parser.each(*line_types) do |request|
-        @summarizer.group(request)  { |r| request_hasher(r) }
-      end
-    end
-
-    # Select the reports to output and generate them.
-    @output_reports.each do |report|
-      report_location = "#{File.dirname(__FILE__)}/../output/#{report}.rb"
-
-      if File.exist?(report_location)
-        eval File.read(report_location)
-      else
-        puts "\nERROR: Output report #{report} not found!"
-      end
-    end
-  end
-  
-  def analyze_with_virtual_mongrels(files = [])
-    # Walk through al the files given via the arguments.
-    files.each do |log_file|
-      puts "Processing #{@line_types.join(', ')} log lines from #{log_file}..."
-
-      parser = @log_parser_class.new(log_file)
-
-      virtual_mongrels = []
-      
-      line = 0
-
-      parser.each(*line_types) do |request|
-        line += 1
-
-        puts "Number of mongrels: #{virtual_mongrels.length}"
-        puts "Line number: #{line}"
-
-        case request[:type]   
-          when :started
-            puts 'Spawned new virtual mongrel'
-            new_mongrel = VirtualMongrel.new(:start_line => line, :calculate_database => @guess_database_time, :running_mongrels => virtual_mongrels.length + 1)
-            new_mongrel.group(request)
-            virtual_mongrels << new_mongrel
-          else
-            completed_mongrel = virtual_mongrels.first
-            completed_mongrel.group(request)
-            completed_mongrel.update_running_mongrels(virtual_mongrels.length)
-            completed_mongrel.save
-        end
-        
-        keep_virtual_mongrels = []
-        
-        virtual_mongrels.each do |mongrel|
-          if mongrel.die_line >= line && mongrel.status == :started
-            keep_virtual_mongrels << mongrel 
-          else
-            puts 'Destroyed virtual mongrel!'
-            puts ""
-
-          end
-        end
-        
-        virtual_mongrels = keep_virtual_mongrels
-            
-      end
-    end
-
-  end
-end
+require 'date'
+require File.dirname(__FILE__) + '/cli/progressbar'
+require File.dirname(__FILE__) + '/cli/bashcolorizer'
+
+require File.dirname(__FILE__) + '/request_log_analyzer/file_format'
+require File.dirname(__FILE__) + '/request_log_analyzer/line_definition'
+require File.dirname(__FILE__) + '/request_log_analyzer/request'
+require File.dirname(__FILE__) + '/request_log_analyzer/log_parser'
+require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/aggregator/summarizer'
+require File.dirname(__FILE__) + '/request_log_analyzer/filter/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/controller'
+require File.dirname(__FILE__) + '/request_log_analyzer/source/base'
+require File.dirname(__FILE__) + '/request_log_analyzer/source/log_file'

data/spec/controller_spec.rb

@@ -4,14 +4,15 @@ describe RequestLogAnalyzer::Controller do
 
   include RequestLogAnalyzerSpecHelper
 
-  it "should include the file format module" do
-    controller = RequestLogAnalyzer::Controller.new(:rails)
-    (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
-  end
+  # it "should include the file format module" do
+  #   controller = RequestLogAnalyzer::Controller.new(:rails)
+  #   (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
+  # end
 
   it "should call the aggregators when run" do
-    controller = RequestLogAnalyzer::Controller.new(:rails)
-    controller << log_fixture(:rails_1x)
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source)
     
     mock_aggregator = mock('aggregator')
     mock_aggregator.should_receive(:prepare).once.ordered
@@ -30,4 +31,10 @@ describe RequestLogAnalyzer::Controller do
     controller.run!
   end
   
+  it "should run well from the command line" do
+    temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
+    system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
+    File.unlink(temp_file)
+  end
+  
 end

data/spec/database_inserter_spec.rb

@@ -8,7 +8,8 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(TestFileFormat, :database => TEST_DATABASE_FILE)
+    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
   end
   
   after(:each) do
@@ -26,7 +27,7 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   it "should create the default table names" do
     @database_inserter.prepare
     @database_inserter.file_format.line_definitions.each do |name, definition|
-      klass = TestFileFormat.const_get("#{name}_line".camelize)
+      klass = SpecFormat.const_get("#{name}_line".camelize)
       klass.column_names.should include('id')
       klass.column_names.should include('lineno')      
       klass.column_names.should include('request_id')      
@@ -36,21 +37,23 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   it "should create the correct fields in the table" do
     @database_inserter.prepare
     
-    TestFileFormat::FirstLine.column_names.should include('request_no')
-    TestFileFormat::LastLine.column_names.should include('request_no')
-    TestFileFormat::TestLine.column_names.should include('test_capture')    
+    SpecFormat::FirstLine.column_names.should include('request_no')
+    SpecFormat::LastLine.column_names.should include('request_no')
+    SpecFormat::TestLine.column_names.should include('test_capture')    
   end
   
 end
 
 describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
-
+  include RequestLogAnalyzerSpecHelper  
+  
   before(:each) do
-    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(TestFileFormat, :database => TEST_DATABASE_FILE)
+    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)    
+    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
     @database_inserter.prepare
         
-    @single = RequestLogAnalyzer::Request.create(TestFileFormat, {:line_type => :first, :request_no => 564})
-    @combined = RequestLogAnalyzer::Request.create(TestFileFormat, 
+    @single = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
+    @combined = RequestLogAnalyzer::Request.create(spec_format, 
                           {:line_type => :first, :request_no  => 564},
                           {:line_type => :test, :test_capture => "awesome"},
                           {:line_type => :test, :test_capture => "indeed"},                                                    
@@ -62,19 +65,19 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
   end 
   
   it "should insert a record in the relevant table" do
-    TestFileFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564))
+    SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564))
     @database_inserter.aggregate(@single)
   end
   
   it "should insert records in all relevant tables" do
-    TestFileFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
-    TestFileFormat::TestLine.should_receive(:create!).twice
-    TestFileFormat::LastLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
+    SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
+    SpecFormat::TestLine.should_receive(:create!).twice
+    SpecFormat::LastLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
     @database_inserter.aggregate(@combined)
   end
   
   it "should log a warning in the warnings table" do
-    TestFileFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
+    SpecFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
     @database_inserter.warning(:test_warning, "Testing the warning system", 12)
   end
   

data/spec/line_definition_spec.rb

@@ -1,12 +1,12 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::FileFormat::LineDefinition do
+describe RequestLogAnalyzer::LineDefinition, :parsing do
   
   before(:each) do
-    @line_definition = RequestLogAnalyzer::FileFormat::LineDefinition.new(:test, {
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
       :teaser   => /Testing /,
       :regexp   => /Testing (\w+), tries\: (\d+)/,
-      :captures => [{:what => :string}, {:tries => :integer}]
+      :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
     })
   end
   
@@ -31,4 +31,94 @@ describe RequestLogAnalyzer::FileFormat::LineDefinition do
   it "should return a hash with :line_type set" do
      @line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
    end
-end
+end
+
+describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
+  before(:each) do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+      :teaser   => /Anonymize /,
+      :regexp   => /Anonymize (\w+)!/,
+      :captures => [{ :name => :what, :type => :string }]
+    })
+  end 
+  
+  it "should return nil if the teaser does not match" do
+    @line_definition.anonymize("Nonsense").should be_nil
+  end
+  
+  it "should return nil if no teaser exists and the regexp doesn't match" do
+    line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize!/, :captures => []})
+
+    line_definition.anonymize('nonsense').should be_nil
+  end
+  
+  it "should return itself if only the teaser matches" do
+    @line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
+  end  
+  
+  it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
+    @line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
+  end
+  
+  it "should return a string if the line matches" do
+    @line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
+  end
+  
+  it "should not anonymize :what" do
+    @line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
+  end  
+end
+
+describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
+
+  it "should anonymize completely if anonymize is true" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
+
+    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
+  end    
+    
+  it "should anonymize a URL" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
+    
+    @line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
+  end
+  
+  it "should anonymize an IP address" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
+    
+    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
+  end 
+  
+  it "should anonymize completely if the anonymizer is unknown" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
+
+    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
+  end
+  
+  it "should anonymize an integer slightly" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
+
+    @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
+  end
+  
+  it "should anonymize an integer slightly" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
+
+    @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
+  end
+  
+  it "should anonymize an double slightly" do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
+
+    @line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
+  end
+   
+end

data/spec/log_parser_spec.rb

@@ -5,18 +5,18 @@ describe RequestLogAnalyzer::LogParser, :single_line_requests do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(TestFileFormat)
+    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
   end
   
   it "should have line definitions" do
     @log_parser.file_format.line_definitions.should_not be_empty
   end
   
-  it "should have include the language specific hooks in the instance, not in the class" do
-    metaclass = (class << @log_parser; self; end)
-    metaclass.ancestors.should include(TestFileFormat::LogParser)
-    @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
-  end
+  # it "should have include the language specific hooks in the instance, not in the class" do
+  #   metaclass = (class << @log_parser; self; end)
+  #   metaclass.ancestors.should include(TestFileFormat::LogParser)
+  #   @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
+  # end
   
   it "should parse a stream and find valid requests" do
     io = File.new(log_fixture(:test_file_format), 'r')
@@ -38,7 +38,7 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(TestFileFormat, :combined_requests => true)
+    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
   end
   
   it "should have multiple line definitions" do
@@ -74,19 +74,17 @@ describe RequestLogAnalyzer::LogParser, :warnings do
   include RequestLogAnalyzerSpecHelper
     
   it "should warn about teaser matching problems" do
-    @log_parser = RequestLogAnalyzer::LogParser.new(TestFileFormat)
+    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
     @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
     @log_parser.parse_file(log_fixture(:test_file_format))
   end
   
   it "should warn about unmatching request headers and footers" do
-    @log_parser = RequestLogAnalyzer::LogParser.new(TestFileFormat, :combined_requests => true)
+    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
     
     @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times    
     @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times    
     @log_parser.should_not_receive(:handle_request)    
     @log_parser.parse_file(log_fixture(:test_order))    
-
   end  
-  
 end