lackac-request-log-analyzer 0.1.3

data/lib/base/record_inserter.rb

@@ -0,0 +1,139 @@
+require 'rubygems'
+require 'sqlite3'
+
+module Base
+  
+  # Set of functions that can be used to easily log requests into a SQLite3 Database.
+  class RecordInserter
+    
+    attr_reader :database
+    attr_reader :current_request
+    attr_reader :warning_count
+    
+    # Initializer
+    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
+    def initialize(db_file, options = {})
+      @database = SQLite3::Database.new(db_file)
+      @insert_statements = nil
+      @warning_count = 0
+      create_tables_if_needed!
+
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+        
+    # Calculate the database durations of the requests currenty in the database.
+    # Used if a logfile does contain any database durations.
+    def calculate_db_durations!
+      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
+    end
+    
+    # Insert a batch of loglines into the database.
+    # Function prepares insert statements, yeilds and then closes and commits.
+    def insert_batch(&block)
+      @database.transaction
+      prepare_statements!
+      block.call(self)
+      close_prepared_statements!
+      @database.commit
+    rescue Exception => e
+      puts e.message
+      @database.rollback
+    end
+    
+    def insert_warning(line, warning)
+      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
+      @warning_count += 1
+    end
+        
+    # Insert a request into the database.
+    # def insert(request, close_statements = false)
+    #   raise 'No insert defined for this log file type'
+    # end
+    
+    # Insert a batch of files into the database.
+    # <tt>db_file</tt> The filename of the database file to use.
+    # Returns the created database.
+    def self.insert_batch_into(db_file, options = {}, &block)
+      db = self.new(db_file)
+      db.insert_batch(&block)
+      return db
+    end    
+    
+    def count(type)
+      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
+    end
+        
+    protected
+    
+    # Prepare insert statements.
+    def prepare_statements!
+      @insert_statements = {
+        :started => @database.prepare("
+            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
+                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
+                                  
+        :failed => @database.prepare("
+            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
+                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
+                                 
+        :completed => @database.prepare("
+            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
+                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
+      }
+    end
+    
+    # Close all prepared statments
+    def close_prepared_statements!
+      @insert_statements.each { |key, stmt| stmt.close }
+    end
+
+    # Create the needed database tables if they don't exist.
+    def create_tables_if_needed!
+      
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS started_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,
+          timestamp DATETIME NOT NULL, 
+          controller VARCHAR(255) NOT NULL, 
+          action VARCHAR(255) NOT NULL,
+          method VARCHAR(6) NOT NULL,          
+          ip VARCHAR(6) NOT NULL
+        )
+      ");
+
+      @database.execute("
+          CREATE TABLE IF NOT EXISTS failed_requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            line INTEGER NOT NULL,            
+            started_request_id INTEGER,        
+            error VARCHAR(255),    
+            exception_string VARCHAR(255),
+            stack_trace TEXT
+          )      
+      ");
+
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS completed_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,          
+          started_request_id INTEGER,
+          url VARCHAR(255) NOT NULL,
+          hashed_url VARCHAR(255),
+          status INTEGER NOT NULL,
+          duration FLOAT,
+          rendering_time FLOAT,
+          database_time FLOAT
+        )
+      ");    
+      
+      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,       
+          warning VARCHAR(255) NOT NULL
+        )
+      ");
+    end
+
+  end
+end

data/lib/base/summarizer.rb

@@ -0,0 +1,71 @@
+module Base
+  
+  # Functions to summarize an array of requets.
+  # Can calculate request counts, duratations, mean times etc. of all the requests given.
+  class Summarizer
+    attr_reader :actions
+    attr_reader :errors
+    attr_reader :request_count
+    attr_reader :request_time_graph
+    attr_reader :first_request_at
+    attr_reader :last_request_at
+    attr_reader :methods
+
+    attr_accessor :blocker_duration
+    DEFAULT_BLOCKER_DURATION = 1.0
+   
+    # Initializer. Sets global variables
+    # Options
+    def initialize(options = {})
+      @actions  = {}
+      @blockers = {}
+      @errors   = {}
+      @request_count = 0
+      @blocker_duration = DEFAULT_BLOCKER_DURATION
+      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+      @methods = {:GET => 0, :POST => 0, :PUT => 0, :DELETE => 0}
+      
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+   
+    # Check if any of the request parsed had a timestamp.
+    def has_timestamps?
+      @first_request_at
+    end
+   
+    # Calculate the duration of a request
+    # Returns a DateTime object if possible, 0 otherwise.
+    def duration
+      (@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).ceil : 0
+    end
+    
+    # Check if the request time graph usable data.
+    def request_time_graph?
+      @request_time_graph.uniq != [0] && duration > 0
+    end
+
+    # Return a list of requests sorted on a specific action field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
+    def sort_actions_by(field, min_count = nil)
+      actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
+      actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_blockers_by(field, min_count = @blocker_duration)
+      blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
+      blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_errors_by(field, min_count = nil)
+      errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
+      errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+  end
+end 

data/lib/bashcolorizer.rb

@@ -0,0 +1,60 @@
+# Colorize a text output with the given color if.
+# <tt>text</tt> The text to colorize.
+# <tt>color_code</tt> The color code string to set
+# <tt>color</tt> Does not color if false. Defaults to ($arguments && $arguments[:colorize])
+def colorize(text, color_code, color = $arguments && $arguments[:colorize])
+  color ? "#{color_code}#{text}\e[0m" : text
+end
+
+# Draw a red line of text
+def red(text)
+  colorize(text, "\e[31m")
+end
+
+# Draw a Green line of text
+def green(text)
+  colorize(text, "\e[32m")
+end
+
+# Draw a Yellow line of text
+def yellow(text)
+  colorize(text, "\e[33m")
+end
+
+# Draw a Yellow line of text
+def blue(text)
+  colorize(text, "\e[34m")
+end
+
+def white(text)
+  colorize(text, "\e[37m")
+end
+
+
+#STYLE = {
+#      :default    =>    “33[0m”,
+#       # styles
+#       :bold       =>    “33[1m”,
+#       :underline  =>    “33[4m”,
+#       :blink      =>    “33[5m”,
+#       :reverse    =>    “33[7m”,
+#       :concealed  =>    “33[8m”,
+#      # font colors
+#       :black      =>    “33[30m”,
+#       :red        =>    “33[31m”,
+#       :green      =>    “33[32m”,
+#       :yellow     =>    “33[33m”,
+#       :blue       =>    “33[34m”,
+#       :magenta    =>    “33[35m”,
+#       :cyan       =>    “33[36m”,
+#       :white      =>    “33[37m”,
+#       # background colors
+#       :on_black   =>    “33[40m”,
+#       :on_red     =>    “33[41m”,
+#       :on_green   =>    “33[42m”,
+#       :on_yellow  =>    “33[43m”,
+#       :on_blue    =>    “33[44m”,
+#       :on_magenta =>    “33[45m”,
+#       :on_cyan    =>    “33[46m”,
+#       :on_white   =>    “33[47m” }
+#

data/lib/command_line/arguments.rb

@@ -0,0 +1,129 @@
+require "#{File.dirname(__FILE__)}/flag"
+require "#{File.dirname(__FILE__)}/exceptions"
+
+# Module used to parse commandline arguments
+module CommandLine
+  
+  # Parse argument lists and return an argument object containing all set flags and switches.
+  class Arguments
+    
+    FLAG_REGEXP = /^--?[A-z0-9]/
+    
+    attr_reader :flag_definitions
+    
+    attr_reader :flags
+    attr_reader :files
+    attr_reader :command
+    
+    attr_accessor :required_files
+    
+    # Initializer.
+    # <tt>arguments</tt> The arguments which are going to be parsed (defaults to $*).
+    def initialize(arguments = $*, &block)
+      @arguments = arguments
+      @flag_definitions = {}
+      @begins_with_command = false
+    end
+    
+    # Parse a list of arguments. Intatiates a Argument object with the given arguments and yeilds
+    # it so that flags and switches can be set by the application.
+    # <tt>arguments</tt> The arguments which are going to be parsed (defaults to $*).
+    # Returns the arguments object.parse!
+    def self.parse(arguments = $*, &block)
+      cla = Arguments.new(arguments)
+      yield(cla)
+      return cla.parse!
+    end
+    
+    # Handle argument switches for the application
+    # <tt>switch</tt> A switch symbol like :fast
+    # <tt>switch_alias</tt> An short alias for the same switch (:f).
+    def switch(switch, switch_alias = nil)
+      return self.flag(switch, :alias => switch_alias, :expects => nil)
+    end
+    
+    # Handle argument flags for the application
+    # <tt>flag</tt> A flag symbol like :fast
+    # Options
+    # * <tt>:expects</tt> Expects a value after the flag
+    def flag(flag, options)
+      options[:expects] = String unless options.has_key?(:expects)
+      argument = Flag.new(flag, options)
+      @flag_definitions[argument.to_argument]  = argument
+      @flag_definitions[argument.to_alias]     = argument if argument.has_alias?
+      return argument
+    end
+    
+    # If called argument list must begin with a command.
+    # <tt>begins_w_command</tt> Defaults to true.
+    def begins_with_command!(begins_w_command=true)
+      @begins_with_command = begins_w_command
+    end
+    
+    # Unknown flags will be silently ignored.
+    # <tt>ignore</tt> Defaults to true.
+    def ignore_unknown_flags!(ignore = true)
+      @ignore_unknown_flags = ignore
+    end
+  
+    def [](name)
+      return flags[name.to_s.gsub(/_/, '-').to_sym]
+    end
+  
+    # Parse the flags and switches set by the application.
+    # Returns an arguments object containing the flags and switches found in the commandline.
+    def parse!
+      @flags = {}
+      @files = []
+      
+      i = 0
+      while @arguments.length > i do
+        arg = @arguments[i]
+        if FLAG_REGEXP =~ arg
+          if @flag_definitions.has_key?(arg)
+            flag = @flag_definitions[arg]
+            if flag.expects_argument?
+              
+              if @arguments.length > (i + 1) && @arguments[i + 1]
+                @flags[flag.name] = @arguments[i + 1]
+                i += 1
+              else
+                raise CommandLine::FlagExpectsArgument.new(arg)
+              end              
+              
+            else
+              @flags[flag.name] = true
+            end
+          else
+            raise CommandLine::UnknownFlag.new(arg) unless @ignore_unknown_flags
+          end
+        else
+          if @begins_with_command && @command.nil?
+            @command = arg
+          else
+            @files << arg
+          end
+        end
+        i += 1        
+      end
+      
+      check_parsed_arguments!
+      
+      return self
+    end
+    
+    # Check if the parsed arguments meet their requirements.
+    # Raises CommandLineexception on error.
+    def check_parsed_arguments!
+      if @begins_with_command && @command.nil? 
+        raise CommandLine::CommandMissing.new
+      end
+
+      if @required_files && @files.length < @required_files
+        raise CommandLine::FileMissing.new("You need at least #{@required_files} files")
+      end
+
+    end
+  end
+
+end

data/lib/command_line/exceptions.rb

@@ -0,0 +1,37 @@
+module CommandLine
+  
+  # Commandline parsing errors and exceptions
+  class Error < Exception
+  end
+
+  # Missing a required flag
+  class FlagMissing < CommandLine::Error
+  end
+
+  # Missing a required file
+  class FileMissing < CommandLine::Error
+  end
+
+  # Missing a required flag argument
+  class FlagExpectsArgument < CommandLine::Error
+    def initialize(flag)
+      super("#{flag} expects an argument!")
+    end    
+  end
+
+  # Missing a required command
+  class CommandMissing < CommandLine::Error
+    def initialize(msg = "A command is missing")
+      super(msg)
+    end
+    
+  end
+
+  # Encountered an unkown flag
+  class UnknownFlag < CommandLine::Error
+    def initialize(flag)
+      super("#{flag} not recognized as a valid flag!")
+    end
+  end
+  
+end

data/lib/command_line/flag.rb

@@ -0,0 +1,51 @@
+module CommandLine
+  
+  # Argument flag handling.
+  class Flag
+    
+    attr_reader :name
+    attr_reader :alias
+    attr_reader :argument
+    
+    # Initialize new Flag
+    # <tt>name</tt> The name of the flag
+    # <tt>definition</tt> The definition of the flag.
+    def initialize(name, definition)
+      @name     = name.to_s.gsub(/_/, '-').to_sym
+      @alias    = definition[:alias].to_sym if definition[:alias]
+      @required = definition.has_key?(:required) && definition[:required] == true
+      @argument = definition[:expects] if definition[:expects]
+    end
+    
+    # Argument representation of the flag (--fast)
+    def to_argument
+      "--#{@name}"
+    end
+    
+    # Argument alias representation of the flag (-f)
+    def to_alias
+      "-#{@alias}"
+    end
+        
+    # Check if flag has an alias
+    def has_alias?
+      !@alias.nil?
+    end
+    
+    # Check if flag is optional
+    def optional?
+      !@required
+    end
+    
+    # Check if flag is required
+    def required?
+      @required
+    end
+    
+    # Check if flag expects an argument (Are you talking to me?)
+    def expects_argument?
+      !@argument.nil?
+    end
+  end  
+  
+end