request-log-analyzer 1.0.2

data/spec/log_processor_spec.rb

@@ -0,0 +1,57 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require 'request_log_analyzer/log_processor'
+
+describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
+
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
+    @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
+  end
+  
+  it "should keep a junk line if :keep_junk_lines is true" do
+    @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
+  end
+  
+  it "should remove a junk line" do
+    @log_anonymizer.anonymize_line("junk line\n").should be_empty
+  end
+
+  it "should keep a teaser line intact" do
+    @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
+  end
+  
+  it "should discard a teaser line if discard_teaser_line is true" do
+    @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
+  end
+  
+  it "should keep a matching line intact if no anonymizing is declared" do
+    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
+  end  
+
+  it "should anonymize values completely if requested" do
+    @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
+  end  
+  
+  it "should anonymize values slightly if requested" do
+    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
+  end
+end
+
+describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
+
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_stripper = RequestLogAnalyzer::LogProcessor.new(spec_format, :strip, {})
+  end
+  
+  it "should remove a junk line" do
+    @log_stripper.strip_line("junk line\n").should be_empty
+  end
+
+  it "should keep a teaser line intact" do
+    @log_stripper.strip_line("processing 1234\n").should be_empty
+  end
+end

data/spec/merb_format_spec.rb

@@ -0,0 +1,38 @@
+# require File.dirname(__FILE__) + '/spec_helper'
+# 
+# describe RequestLogAnalyzer::LogParser, "Merb" do
+#   include RequestLogAnalyzerSpecHelper
+#   
+#   before(:each) do
+#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
+#   end
+#   
+#   it "should have a valid language definitions" do
+#     @log_parser.file_format.should be_valid
+#   end
+#   
+#   it "should parse a stream and find valid requests" do
+#     File.open(log_fixture(:merb), 'r') do |io|
+#       @log_parser.parse_io(io) do |request| 
+#         request.should be_kind_of(RequestLogAnalyzer::Request)
+#       end
+#     end
+#   end
+#
+#   it "should find 11 completed requests" do
+#     @log_parser.should_receive(:handle_request).exactly(11).times
+#     @log_parser.parse_file(log_fixture(:merb))
+#   end
+#   
+#   it "should parse all details from a request correctly" do
+#     request = nil
+#     @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
+#     
+#     request.should be_completed
+#     request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008')
+#     request[:dispatch_time].should == 0.243424
+#     request[:after_filters_time].should == 6.9e-05
+#     request[:before_filters_time].should == 0.213213
+#     request[:action_time].should == 0.241652
+#   end
+# end

data/spec/rails_format_spec.rb

@@ -0,0 +1,76 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::LogParser, "Rails" do
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
+  end
+  
+  it "should have a valid language definitions" do
+    @log_parser.file_format.should be_valid
+  end
+  
+  it "should parse a stream and find valid requests" do
+    io = File.new(log_fixture(:rails_1x), 'r')
+    @log_parser.parse_io(io) do |request| 
+      request.should be_kind_of(RequestLogAnalyzer::Request)
+    end
+    io.close
+  end  
+  
+  it "should find 4 completed requests" do
+    @log_parser.should_not_receive(:warn)  
+    @log_parser.should_receive(:handle_request).exactly(4).times
+    @log_parser.parse_file(log_fixture(:rails_1x))
+  end  
+  
+  it "should parse a Rails 2.2 request properly" do
+    @log_parser.should_not_receive(:warn)
+    @log_parser.parse_file(log_fixture(:rails_22)) do |request|
+      request.should =~ :processing
+      request.should =~ :completed  
+    
+      request[:controller].should == 'PageController'
+      request[:action].should     == 'demo'
+      request[:url].should        == 'http://www.example.coml/demo'    
+      request[:status].should     == 200
+      request[:duration].should   == 0.614
+      request[:db].should         == 0.031
+      request[:view].should       == 0.120
+    end
+  end
+  
+  it "should parse a syslog file with prefix correctly" do
+    @log_parser.should_not_receive(:warn)    
+    @log_parser.parse_file(log_fixture(:syslog_1x)) do |request| 
+      
+      request.should be_completed
+      
+      request[:controller].should == 'EmployeeController'
+      request[:action].should     == 'index'
+      request[:url].should        == 'http://example.com/employee.xml'    
+      request[:status].should     == 200
+      request[:duration].should   == 0.21665
+      request[:db].should         == 0.0
+      request[:view].should       == 0.00926
+    end
+  end
+  
+  it "should parse cached requests" do
+    @log_parser.should_not_receive(:warn)
+    @log_parser.parse_file(log_fixture(:rails_22_cached)) do |request| 
+      request.should be_completed
+      request =~ :cache_hit
+    end  
+  end
+  
+  it "should detect unordered requests in the logs" do
+    @log_parser.should_not_receive(:handle_request)
+    # the first Processing-line will not give a warning, but the next one will
+    @log_parser.should_receive(:warn).with(:unclosed_request, anything).once
+    # Both Completed ;ines will give a warning
+    @log_parser.should_receive(:warn).with(:no_current_request, anything).twice
+    @log_parser.parse_file(log_fixture(:rails_unordered))
+  end  
+end

data/spec/request_spec.rb

@@ -0,0 +1,72 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::Request, :incomplete_request do
+  
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
+    @incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
+  end
+  
+  it "should be single if only one line has been added" do
+    @incomplete_request.should_not be_empty
+  end
+  
+  it "should not be a completed request" do
+    @incomplete_request.should_not be_completed
+  end  
+  
+  it "should take the line type of the first line as global line_type" do
+    @incomplete_request.lines[0][:line_type].should == :test
+    @incomplete_request.should =~ :test
+  end
+  
+  it "should return the first field value" do
+    @incomplete_request[:test_capture].should == 'awesome!'
+  end
+  
+  it "should return nil if no such field is present" do
+    @incomplete_request[:nonexisting].should be_nil
+  end
+end  
+
+
+describe RequestLogAnalyzer::Request, :completed_request do
+  
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @completed_request = RequestLogAnalyzer::Request.new(spec_format)
+    @completed_request << { :line_type => :first, :lineno =>  1, :name => 'first line!' }    
+    @completed_request << { :line_type => :test,  :lineno =>  4, :test_capture => 'testing' }        
+    @completed_request << { :line_type => :test,  :lineno =>  7, :test_capture => 'testing some more' }            
+    @completed_request << { :line_type => :last,  :lineno => 10, :time => 0.03 }
+  end
+  
+  it "should not be empty when multiple liness are added" do
+    @completed_request.should_not be_empty
+  end
+  
+  it "should be a completed request" do
+    @completed_request.should be_completed
+  end
+  
+  it "should recognize all line types" do
+    [:first, :test, :last].each { |type| @completed_request.should =~ type }
+  end
+  
+  it "should detect the correct field value" do
+    @completed_request[:name].should == 'first line!'
+    @completed_request[:time].should == 0.03
+  end
+  
+  it "should detect the first matching field value" do  
+    @completed_request.first(:test_capture).should == 'testing'
+  end
+  
+  it "should detect the every matching field value" do  
+    @completed_request.every(:test_capture).should == ['testing', "testing some more"]
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,67 @@
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'spec'
+require 'request_log_analyzer'
+
+module RequestLogAnalyzerSpecHelper
+  
+  def format_file(format)
+    File.dirname(__FILE__) + "/file_formats/#{format}.rb"
+  end
+  
+  def spec_format
+    @spec_format ||= begin
+      require format_file(:spec_format)
+      SpecFormat.new
+    end
+  end
+      
+  def log_fixture(name)
+    File.dirname(__FILE__) + "/fixtures/#{name}.log"
+  end
+  
+  def request(fields, format = TestFileFormat)
+    if fields.kind_of?(Array)
+      RequestLogAnalyzer::Request.create(format, *fields)
+    else
+      RequestLogAnalyzer::Request.create(format, fields)
+    end
+  end
+  
+end
+
+module TestFileFormat
+  
+  module Summarizer
+    def self.included(base)
+      # monkey patching for summarizer here :-)
+    end
+  end
+  
+  module LogParser
+    def self.included(base)
+      # monkey patching for log parser here :-)
+    end
+  end
+  
+  LINE_DEFINITIONS = {
+    :first => {
+      :header => true,
+      :teaser => /processing /,
+      :regexp => /processing request (\d+)/,
+      :captures => [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
+    },
+    :test => {
+      :teaser => /testing /,
+      :regexp => /testing is (\w+)/,
+      :captures => [{ :name => :test_capture, :type => :string, :anonymize => true}]
+    }, 
+    :last => {
+      :footer => true,
+      :teaser => /finishing /,
+      :regexp => /finishing request (\d+)/,
+      :captures => [{ :name => :request_no, :type => :integer}]
+    }
+  }
+end

data/spec/summarizer_spec.rb

@@ -0,0 +1,9 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require 'request_log_analyzer/aggregator/summarizer'
+
+describe RequestLogAnalyzer::Aggregator::Summarizer do
+  
+  before(:each) do
+  end
+
+end

data/tasks/github-gem.rake

@@ -0,0 +1,177 @@
+require 'rubygems'
+require 'rake'
+require 'rake/tasklib'
+require 'date'
+
+module Rake 
+  
+  class GithubGem < TaskLib
+    
+    attr_accessor :name
+    attr_accessor :specification
+   
+    def self.define_tasks!
+      gem_task_builder = Rake::GithubGem.new      
+      gem_task_builder.register_all_tasks!
+    end
+    
+
+    def initialize
+      reload_gemspec!
+    end
+
+    def register_all_tasks!
+      namespace(:gem) do
+        desc "Updates the file lists for this gem"
+        task(:manifest) { manifest_task }
+        
+        desc "Builds a ruby gem for #{@name}"
+        task(:build => [:manifest]) { build_task }
+        
+        desc "Installs the ruby gem for #{@name} locally"
+        task(:install => [:build]) { install_task }
+        
+        desc "Uninstalls the ruby gem for #{@name} locally"
+        task(:uninstall) { uninstall_task }             
+        
+        desc "Releases a new version of #{@name}"
+        task(:release) { release_task } 
+      end
+    end
+    
+
+    
+    protected 
+
+    def reload_gemspec!
+      raise "No gemspec file found!" if gemspec_file.nil?      
+      spec = File.read(gemspec_file)
+      @specification = eval(spec)
+      @name = specification.name  
+    end
+
+    def run_command(command)
+      lines = []
+      IO.popen(command) { |f| lines = f.readlines }
+      return lines
+    end
+    
+    def git_modified?(file)
+      return !run_command('git status').detect { |line| Regexp.new(Regexp.quote(file)) =~ line }.nil?
+    end
+    
+    def git_commit_file(file, message, branch = nil)
+      verify_current_branch(branch) unless branch.nil?
+      if git_modified?(file)
+        sh "git add #{file}"
+        sh "git commit -m \"#{message}\""
+      else
+        raise "#{file} is not modified and cannot be committed!"
+      end
+    end
+    
+    def git_create_tag(tag_name, message)
+      sh "git tag -a \"#{tag_name}\" -m \"#{message}\""
+    end
+    
+    def git_push(remote = 'origin', branch = 'master', options = [])
+      verify_clean_status(branch)
+      options_str = options.map { |o| "--#{o}"}.join(' ')
+      sh "git push #{options_str} #{remote} #{branch}"
+    end
+    
+    def gemspec_version=(new_version)
+      spec = File.read(gemspec_file)
+      spec.gsub!(/^(\s*s\.version\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_version}'#{$5}" }
+      spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{Date.today.strftime('%Y-%m-%d')}'#{$5}" }    
+      File.open(gemspec_file, 'w') { |f| f << spec }
+      reload_gemspec!      
+    end
+    
+    def gemspec_date=(new_date)
+      spec = File.read(gemspec_file)
+      spec.gsub!(/^(\s*s\.date\s*=\s*)('|")(.+)('|")(\s*)$/) { "#{$1}'#{new_date.strftime('%Y-%m-%d')}'#{$5}" }    
+      File.open(gemspec_file, 'w') { |f| f << spec }
+      reload_gemspec!       
+    end
+    
+    def gemspec_file 
+      @gemspec_file ||= Dir['*.gemspec'].first
+    end
+    
+    def verify_current_branch(branch)
+      run_command('git branch').detect { |line| /^\* (.+)/ =~ line }
+      raise "You are currently not working in the master branch!" unless branch == $1
+    end
+    
+    def verify_clean_status(on_branch = nil)
+      sh "git fetch"
+      lines = run_command('git status')
+      raise "You don't have the most recent version available. Run git pull first." if /^\# Your branch is behind/ =~ lines[1]
+      raise "You are currently not working in the #{on_branch} branch!" unless on_branch.nil? || (/^\# On branch (.+)/ =~ lines.first && $1 == on_branch)
+      raise "Your master branch contains modifications!" unless /^nothing to commit \(working directory clean\)/ =~ lines.last
+    end
+    
+    def verify_version(new_version)
+      newest_version = run_command('git tag').map { |tag| tag.split(name + '-').last }.compact.map { |v| Gem::Version.new(v) }.max
+      raise "This version number (#{new_version}) is not higher than the highest tagged version (#{newest_version})" if !newest_version.nil? && newest_version >= Gem::Version.new(new_version.to_s)
+    end
+
+    def manifest_task
+      verify_current_branch('master')
+      
+      list = Dir['**/*'].sort
+      list -= [gemspec_file]
+
+      if File.exist?('.gitignore')
+        File.read('.gitignore').each_line do |glob|
+          glob = glob.chomp.sub(/^\//, '')
+          list -= Dir[glob]
+          list -= Dir["#{glob}/**/*"] if File.directory?(glob) and !File.symlink?(glob)
+        end
+      end
+      
+      # update the spec file
+      spec = File.read(gemspec_file)
+      spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
+        assignment = $1
+        bunch = $2 ? list.grep(/^(test.*_test\.rb|spec.*_spec.rb)$/) : list
+        '%s%%w(%s)' % [assignment, bunch.join(' ')]
+      end
+
+      File.open(gemspec_file, 'w') { |f| f << spec }
+      reload_gemspec!
+    end
+    
+    def build_task
+      sh "gem build #{gemspec_file}"
+    end
+    
+    def install_task
+      raise "#{name} .gem file not found" unless File.exist?("#{name}-#{specification.version}.gem")
+      sh "gem install #{name}-#{specification.version}.gem"
+    end
+    
+    def uninstall_task
+      raise "#{name} .gem file not found" unless File.exist?("#{name}-#{specification.version}.gem")
+      sh "gem uninstall #{name}"
+    end    
+    
+    def release_task
+      verify_clean_status('master')
+      verify_version(ENV['VERSION'] || @specification.version)
+      
+      # update gemspec file
+      self.gemspec_version = ENV['VERSION'] if Gem::Version.correct?(ENV['VERSION'])
+      self.gemspec_date    = Date.today
+      manifest_task      
+      git_commit_file(gemspec_file, "Updated #{gemspec_file} for release of version #{@specification.version}") if git_modified?(gemspec_file)
+
+      # create tag and push changes
+      git_create_tag("#{@name}-#{@specification.version}", "Tagged version #{@specification.version}")
+      git_push('origin', 'master', [:tags])
+    end
+  end
+end
+
+Rake::GithubGem.define_tasks!