request-log-analyzer 1.0.2

data/DESIGN

@@ -0,0 +1,14 @@
+Request-log-analyzer is set up like a simple pipe and filter system.
+
+This allows you to easily add extra reports, filters and outputs.
+
+1) Build pipeline.
+                             -> Aggregator  (database)
+  Source -> Filter -> Filter -> Aggregator  (summary report)
+                             -> Aggregator  (...)
+
+2) Start chunk producer and push chunks through pipeline.
+  Controller.start
+
+3) Gather output from pipeline.
+  Controller.report

data/HACKING

@@ -0,0 +1,7 @@
+HACKING on r-l-a
+----------------
+
+- See DESIGN for the basic internal design of r-l-a
+- See http://wiki.github.com/wvanbergen/request-log-analyzer/development for
+  more information about developing
+- Contact willem AT vanbergen DOT org for any questions

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Willem van Bergen / Bart ten Brinke
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.textile

@@ -0,0 +1,36 @@
+h1. Request-log-analyzer
+
+This is a simple command line tool to analyze request log files of both Rails and
+Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
+
+* Analyzes Rails log files (all versions)
+* Can combine multiple files (handy if you are using logrotate)
+* Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and states, Rails action cache statistics, etc.) ("Sample output":http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
+* Low memory footprint (server-safe)
+* Fast
+* MIT licensed
+
+h2. Installation
+
+<pre>
+$ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
+</pre>
+
+To get the best results out of request-log-analyzer, make sure to 
+"set up logging correctly":http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
+for your application.
+
+h2. Usage
+
+To analyze a log file and produce a performance report, run request-log-analyzer like this:
+
+<pre>
+$ request-log-analyzer log/production.log
+</pre>
+
+For more details and available command line options, see the "project's wiki":http://wiki.github.com/wvanbergen/request-log-analyzer/basic-usage
+
+h2. Additional information
+
+* "Project wiki at GitHub":http://wiki.github.com/wvanbergen/request-log-analyzer
+* "wvanbergen's blog posts":http://techblog.floorplanner.com/tag/request-log-analyzer/

data/Rakefile

@@ -0,0 +1,5 @@
+Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
+ 
+desc 'Default: run RSpec for request-log-analyzer.'
+task :default => :spec
+

data/bin/request-log-analyzer

@@ -0,0 +1,123 @@
+#!/usr/bin/ruby
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
+require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
+
+def terminal_width(default = 81)
+  IO.popen('stty -a') do |pipe|
+    column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+    width = column_line ? $1.to_i : default
+  end
+rescue 
+  default  
+end
+
+# Parse the arguments given via commandline
+begin
+  arguments = CommandLine::Arguments.parse do |command_line|
+
+    command_line.command(:install) do |install|
+      install.parameters = 1
+    end
+    
+    command_line.command(:strip) do |strip|
+      strip.minimum_parameters = 1
+      strip.option(:format, :alias => :f, :default => 'rails')      
+      strip.option(:output, :alias => :o)
+      strip.switch(:discard_teaser_lines, :t)
+      strip.switch(:keep_junk_lines, :j)      
+    end
+    
+    command_line.command(:anonymize) do |anonymize|
+      anonymize.minimum_parameters = 1
+      anonymize.option(:format, :alias => :f, :default => 'rails')            
+      anonymize.option(:output, :alias => :o)
+      anonymize.switch(:discard_teaser_lines, :t)
+      anonymize.switch(:keep_junk_lines, :j)      
+    end
+    
+    command_line.option(:format, :alias => :f, :default => 'rails')
+    command_line.option(:file, :alias => :e)
+    command_line.switch(:assume_correct_order)
+    
+    command_line.option(:aggregator, :alias => :a, :multiple => true)
+    command_line.option(:database, :alias => :d)    
+
+    # filtering options
+    command_line.option(:select, :multiple => true, :parameters => 2)
+    command_line.option(:reject, :multiple => true, :parameters => 2)
+    command_line.option(:after)
+    command_line.option(:before)    
+    
+    command_line.switch(:boring, :b)
+    command_line.option(:report_width, :default => terminal_width - 1)
+    
+    command_line.switch(:debug)
+    
+    command_line.minimum_parameters = 1    
+  end
+  
+rescue CommandLine::Error => e
+  puts "ARGUMENT ERROR: " + e.message if e.message
+  puts
+  puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
+  puts
+  puts "Input options:"
+  puts "  --format <format>, -f:     Uses the specified log file format. Defaults to rails."
+  puts "  --after <date>             Only consider requests from <date> or later."
+  puts "  --before <date>            Only consider requests before <date>."  
+  puts "  --select <field> <value>   Only consider requests where <field> matches <value>."        
+  puts "  --reject <field> <value>   Only consider requests where <field> does not match <value>."    
+  puts
+  puts "Output options:"
+  puts "  --boring, -b               Output reports without ASCII colors."
+  puts "  --database <filename>, -d: Creates an SQLite3 database of all the parsed request information."
+  puts "  --debug                    Print debug information while parsing."
+  puts "  --file <filename>          Output to file."
+  puts 
+  puts "Examples:"
+  puts "  request-log-analyzer development.log"
+  puts "  request-log-analyzer -b mongrel.0.log mongrel.1.log mongrel.2.log "
+  puts "  request-log-analyzer --format merb -d requests.db production.log"
+  puts
+  puts "To install rake tasks in your Rails application, "
+  puts "run the following command in your application's root directory:"
+  puts
+  puts "  request-log-analyzer install rails"
+  exit(0) 
+end
+
+def install_rake_tasks(install_type)
+  if install_type == 'rails'
+    require 'ftools'
+    if File.directory?('./lib/tasks/')
+      File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
+      puts "Installed rake tasks."
+      puts "To use, run: rake log:analyze"
+    else
+      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
+      puts "Installation aborted."
+    end
+  else
+    raise "Cannot perform this install type! (#{install_type})"
+  end
+end
+
+
+case arguments.command
+when :install
+  install_rake_tasks(arguments.parameters[0])
+when :strip
+  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'  
+  RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
+when :anonymize
+  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'    
+  RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!  
+else 
+  puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke - Version 1.0\n\n"  
+  
+  # Run the request_log_analyzer!
+  RequestLogAnalyzer::Controller.build(arguments, terminal_width).run!
+
+  puts 
+  puts "Thanks for using request-log-analyzer"
+end

data/lib/cli/bashcolorizer.rb

@@ -0,0 +1,60 @@
+# Colorize a text output with the given color if.
+# <tt>text</tt> The text to colorize.
+# <tt>color_code</tt> The color code string to set
+# <tt>color</tt> Does not color if false. Defaults to false
+def colorize(text, color_code, color = false)
+  color ? "#{color_code}#{text}\e[0m" : text
+end
+
+# Draw a red line of text
+def red(text, color = false)
+  colorize(text, "\e[31m", color)
+end
+
+# Draw a Green line of text
+def green(text, color = false)
+  colorize(text, "\e[32m", color)
+end
+
+# Draw a Yellow line of text
+def yellow(text, color = false)
+  colorize(text, "\e[33m", color)
+end
+
+# Draw a Yellow line of text
+def blue(text, color = false)
+  colorize(text, "\e[34m", color)
+end
+
+def white(text, color = false)
+  colorize(text, "\e[37m", color)
+end
+
+
+#STYLE = {
+#      :default    =>    “33[0m”,
+#       # styles
+#       :bold       =>    “33[1m”,
+#       :underline  =>    “33[4m”,
+#       :blink      =>    “33[5m”,
+#       :reverse    =>    “33[7m”,
+#       :concealed  =>    “33[8m”,
+#      # font colors
+#       :black      =>    “33[30m”,
+#       :red        =>    “33[31m”,
+#       :green      =>    “33[32m”,
+#       :yellow     =>    “33[33m”,
+#       :blue       =>    “33[34m”,
+#       :magenta    =>    “33[35m”,
+#       :cyan       =>    “33[36m”,
+#       :white      =>    “33[37m”,
+#       # background colors
+#       :on_black   =>    “33[40m”,
+#       :on_red     =>    “33[41m”,
+#       :on_green   =>    “33[42m”,
+#       :on_yellow  =>    “33[43m”,
+#       :on_blue    =>    “33[44m”,
+#       :on_magenta =>    “33[45m”,
+#       :on_cyan    =>    “33[46m”,
+#       :on_white   =>    “33[47m” }
+#

data/lib/cli/command_line_arguments.rb

@@ -0,0 +1,301 @@
+module CommandLine
+
+  class Option
+
+    attr_reader :name, :alias
+    attr_reader :parameter_count
+    attr_reader :default_value
+
+    # Rewrites a command line keyword by replacing the underscores with dashes
+    # <tt>sym</tt> The symbol to rewrite
+    def self.rewrite(sym)
+      sym.to_s.gsub(/_/, '-').to_sym
+    end
+
+    # Initialize new CommandLine::Option
+    # <tt>name</tt> The name of the flag
+    # <tt>definition</tt> The definition of the flag.
+    def initialize(name, definition = {})
+      @name            = CommandLine::Option.rewrite(name)
+      @alias           = definition[:alias].to_sym if definition[:alias]
+      @required        = definition.has_key?(:required) && definition[:required] == true
+      @parameter_count = definition[:parameters] || 1
+      @multiple        = definition[:multiple]   || false
+      @default_value   = definition[:default]    || false
+    end
+
+    def parse(arguments_parser)
+      if @parameter_count == 0
+        return true
+      elsif @parameter_count == 1
+        parameter = arguments_parser.next_parameter
+        raise CommandLine::ParameterExpected, self if parameter.nil?
+        return parameter
+      elsif @parameter_count == :any
+        parameters = []
+        while parameter = arguments_parser.next_parameter && parameter != '--'
+          parameters << parameter
+        end
+        return parameters
+      else
+        parameters = []
+        @parameter_count.times do |n|
+          parameter = arguments_parser.next_parameter
+          raise CommandLine::ParameterExpected, self if parameter.nil?
+          parameters << parameter
+        end
+        return parameters
+      end
+    end
+
+    def =~(test)
+      [@name, @alias].include?(CommandLine::Option.rewrite(test))
+    end
+
+    # Argument representation of the flag (--fast)
+    def to_option
+      "--#{@name}"
+    end
+
+    # Argument alias representation of the flag (-f)
+    def to_alias
+      "-#{@alias}"
+    end
+
+    # Check if flag has an alias
+    def has_alias?
+      !@alias.nil?
+    end
+
+    # Check if flag is required
+    def required?
+      @required
+    end
+    
+    # Check if flag is optional
+    def optional?
+      !@required
+    end
+
+    def multiple?
+      @multiple
+    end
+
+    def has_default?
+      !@default_value.nil?
+    end
+  end
+  
+  class Arguments
+
+    class Definition
+      
+      ENDLESS_PARAMETERS = 99999
+      
+      attr_reader :commands, :options, :parameters
+      
+      def initialize(parent)
+        @parent = parent
+        @options  = {}
+        @commands = {}
+        @parameters = nil
+      end
+      
+      def [](option_name)
+        option_symbol = CommandLine::Option.rewrite(option_name)
+        if the_option = @options.detect { |(name, odef)| odef =~ option_symbol }
+          the_option[1]
+        else
+          raise CommandLine::UnknownOption, option_name
+        end
+      end
+      
+      def minimum_parameters=(count_specifier)
+        @parameters = count_specifier..ENDLESS_PARAMETERS
+      end
+            
+      def parameters=(count_specifier)
+        @parameters = count_specifier
+      end
+      
+      alias :files= :parameters= 
+      
+      def option(name, options = {})
+        clo = CommandLine::Option.new(name, options)
+        @options[clo.name] = clo
+      end
+      
+      def switch(name, switch_alias = nil)
+        option(name, :alias => switch_alias, :parameters => 0)
+      end
+
+      def command(name, &block)
+        command_definition = Definition.new(self)
+        yield(command_definition) if block_given?
+        @commands[CommandLine::Option.rewrite(name)] = command_definition
+      end
+      
+      def has_command?(command)
+        @commands[CommandLine::Option.rewrite(command)]
+      end
+    end
+    
+    OPTION_REGEXP  = /^\-\-([A-z0-9-]+)$/;
+    ALIASES_REGEXP = /^\-([A-z0-9]+)$/
+    
+    attr_reader :definition
+    attr_reader :tokens
+    attr_reader :command, :options, :parameters
+    
+    def self.parse(tokens = $*, &block)
+      cla = Arguments.new
+      cla.define(&block)
+      return cla.parse!(tokens)   
+    end
+    
+    def initialize
+      @tokens = []
+      @definition = Definition.new(self)    
+      @current_definition = @definition        
+    end
+    
+    def define(&block)
+      yield(@definition)
+    end
+    
+    def [](option)
+      if the_option = @options.detect { |(key, value)| key =~ option }
+        the_option[1]
+      else
+        @current_definition[option].default_value
+      end
+    end
+        
+    def next_token
+      @current_token = @tokens.shift
+      return @current_token
+    end
+    
+    def next_parameter
+      parameter_candidate = @tokens.first
+      parameter = (parameter_candidate.nil? || OPTION_REGEXP =~ parameter_candidate || ALIASES_REGEXP =~ parameter_candidate) ? nil : @tokens.shift
+      return parameter
+    end
+    
+    def parse!(tokens)
+      @current_definition = @definition         
+      @first_token = true
+      @tokens      = tokens.clone
+      
+      @options = {}
+      @parameters   = []
+      @command = nil
+      
+      prepare_result!
+      
+      while next_token
+        
+        if @first_token && command_definition = @definition.has_command?(@current_token)
+          @current_definition = command_definition
+          @command = CommandLine::Option.rewrite(@current_token)
+        else
+          case @current_token
+            when ALIASES_REGEXP; handle_alias_expansion($1)
+            when OPTION_REGEXP;  handle_option($1)
+            else;                handle_other_parameter(@current_token)
+          end
+          @first_token = false          
+        end
+          
+      end
+      
+      validate_arguments!
+      
+      return self     
+    end
+    
+    protected
+    
+    def prepare_result!
+      multiple_options = Hash[*@current_definition.options.select { |name, o| o.multiple? }.flatten]
+      multiple_options.each { |name, definition| @options[definition] = [] }
+    end
+    
+    def validate_arguments!
+      if @current_definition.parameters && !(@current_definition.parameters === @parameters.length)
+        raise CommandLine::ParametersOutOfRange.new(@current_definition.parameters, @parameters.length)
+      end
+      
+      required_options = Hash[*@current_definition.options.select { |name, o| o.required? }.flatten]
+      required_options.each do |name, definition|
+        raise CommandLine::RequiredOptionMissing, definition unless self[name] 
+      end
+    end
+    
+    def handle_alias_expansion(aliases)
+      aliases.reverse.scan(/./) do |alias_char|
+        if option_definition = @current_definition[alias_char]
+          @tokens.unshift(option_definition.to_option)
+        else
+          raise CommandLine::UnknownOption, alias_char
+        end
+      end
+    end
+    
+    def handle_other_parameter(parameter)
+      @parameters << parameter
+    end
+    
+    def handle_option(option_name)
+      option_definition = @current_definition[option_name]
+      raise CommandLine::UnknownOption, option_name if option_definition.nil?
+      
+      if option_definition.multiple?
+        @options[option_definition] << option_definition.parse(self)        
+      else
+        @options[option_definition] = option_definition.parse(self)
+      end
+    end
+        
+  end
+  
+  # Commandline parsing errors and exceptions
+  class Error < Exception
+  end
+
+  # Missing a required flag
+  class RequiredOptionMissing < CommandLine::Error
+    def initialize(option)
+      super("You have to provide the #{option.name} option!")
+    end    
+  end
+
+  # Missing a required file
+  class ParametersOutOfRange < CommandLine::Error
+    def initialize(expected, actual)
+      if expected.kind_of?(Range)
+        if expected.end == CommandLine::Arguments::Definition::ENDLESS_PARAMETERS
+          super("The command expected at least #{expected.begin} parameters, but found #{actual}!")
+        else
+          super("The command expected between #{expected.begin} and #{expected.end} parameters, but found #{actual}!")
+        end
+      else
+        super("The command expected #{expected} parameters, but found #{actual}!")
+      end
+    end    
+  end
+
+  # Missing a required flag argument
+  class ParameterExpected < CommandLine::Error
+    def initialize(option)
+      super("The option #{option.inspect} expects a parameter!")
+    end    
+  end
+
+  # Encountered an unkown flag
+  class UnknownOption < CommandLine::Error
+    def initialize(option_identifier)
+      super("#{option_identifier.inspect} not recognized as a valid option!")
+    end
+  end  
+end