rawkx 0.1.0

data/README

@@ -0,0 +1,43 @@
+RawkX - An Extendable Version of the Rawk Log Analyzer
+Author: Peter Zimbelman
+Git Repository: http://github.com/pzimbelman/rawkx
+
+This project is based off the Rawk project (http://rubyforge.org/projects/rawk-the-logs/) and uses much of its original code.
+RawkX is mostly a reorganization of this functionality to make extending the analyzer with different log formats easier.
+
+This gem is designed to allow users to easily extend the Rawk analyzer to have it use a seperate method for parsing.
+This will enable the analyzer to parse and perform calculations on logs of any format, not just Rails logs.
+
+This is done by simply overriding the 'parse_line' method used by RawkX to obtain a key/value pair for tracking times (values) associated with
+their action/request (keys).  Upon doing this, you can call RawkX as you normally would and can get all the useful statistics you would normally
+get against any Rails log.
+
+Also note: by default (no override of parse_line), RawkX has the ability to parse Rails logs as Rawk always has.
+
+Below is a simple example to illustrate using RawkX:
+
+For instance, lets say your Log looked like this:
+Format: time_to_complete action username
+-------------------------------------------------
+3.24 Read user1
+5.0 Write user1
+2.245 Read user2
+3.4 Read user3
+--------------------------------------------------
+
+You could create a simple ruby file which contains the following:
+--------------------------------------------------
+require "rubygems"
+require "rawkx"
+
+class RawkX
+  def parse_line(line)
+    fields = line.split
+    return fields[1], fields[0]
+  end
+end
+RawkX.new
+--------------------------------------------------
+
+this file can be run with the command: ruby <ruby_file_name> -f <log_file_name>
+Running this will produce Rawk results for your logs. you would be using the Actions (Read or Write) as your keys and the time_to_complete as your values

data/lib/line_parser.rb

@@ -0,0 +1,27 @@
+
+module LineParser
+  #Default parse implementation. This is the original Rawk implementation for a Rails log.
+  def parse_line(line)
+     @last_actions ||= {}
+     pid_regexp = /\(pid\:\d+\)/
+     if line.index("Processing ")==0
+        action = line.split[1]
+        pid = line[pid_regexp]
+        @last_actions[pid]=action if pid
+        return
+      end
+      return unless line.index("Completed in")==0
+      #get the pid unless we are forcing url tracking
+      pid = line[pid_regexp]
+      key = pid ? @last_actions[pid] : nil
+
+      time_string = line[/Completed in \d+\.\d+/]
+      time_string = time_string[/\d+\.\d+/] if time_string
+      time = time_string ? time_string.to_f : 0.0
+     
+      #if pids are not specified then we use the url for hashing
+      #the below regexp turns "[http://spongecell.com/calendar/view/bob]" to "/calendar/view"
+      key = (line[/\[\S+\]/].gsub(/\S+\/\/(\w|\.)*/,''))[/\/\w*\/?\w*/] unless key
+      return key, time
+  end
+end

data/lib/rawkx.rb

@@ -0,0 +1,110 @@
+require "#{File.expand_path(File.dirname(__FILE__))}/stat"
+require "#{File.expand_path(File.dirname(__FILE__))}/stat_hash"
+require "#{File.expand_path(File.dirname(__FILE__))}/line_parser"
+
+
+class RawkX
+  include LineParser
+
+  VERSION = "0.1.0"
+  HEADER = "Request                                        Count     Sum     Max  Median     Avg     Min     Std"
+  HELP = "\nRawkX - Extendedable Rawk v#{VERSION}\n"+
+  "Created by Peter Zimbelman\n"+
+  "The foundation of this gem is based off of the Rawk project. This is mostly a refactoring of that gem to make extending it easier\n" +    
+  "The log file is read from standard input unless the -f flag is specified.\n\n"+
+  "The options are as follows:\n\n"+
+  "  -f <filename> Use the specified file instead of standard input.\n\n"+
+  "  -h  Display this help.\n\n"+
+  "  -s <count> Display <count> results in each group of data.\n\n"+
+  "  -w <count> Display the top <count> worst requests.\n\n"
+
+  def initialize
+    @start_time = Time.now
+    build_arg_hash
+    if @arg_hash.keys.include?("h")
+      puts HELP
+    else
+      init_args
+      build_stats
+      print_stats
+    end
+  end
+
+
+  private
+  def build_arg_hash
+    @arg_hash = Hash.new
+    last_key=nil
+    for a in $*
+      if a.index("-")==0 && a.length>1
+        a[1,1000].scan(/[a-z]|\?/).each {|c| @arg_hash[last_key=c]=nil}
+        @arg_hash[last_key] = a[/\d+/] if last_key
+      elsif a.index("-")!=0 && last_key
+        @arg_hash[last_key] = a
+      end
+    end
+  end
+  
+  def init_args
+    @input = $stdin
+    @input = File.new(@arg_hash["f"]) if @arg_hash["f"]
+    @worst_request_length= (@arg_hash["w"] || 20).to_i
+    @sorted_limit = (@arg_hash["s"] || 20).to_i
+    @stat_hash = StatHash.new
+    @total_stat = Stat.new("All Requests")
+    @worst_requests = []
+  end
+
+  def build_stats
+    while line = @input.gets
+      key, time = parse_line(line)
+      next unless key
+      time = time.to_f unless time.is_a? Float
+      @stat_hash.add(key,time)
+      @total_stat.add(time)
+      if is_a_worst_request?(time)
+         add_to_and_reorder_worst_requests([time,line])
+      end
+    end
+  end
+
+  def is_a_worst_request?(time)
+    @worst_requests.length < @worst_request_length || @worst_requests[@worst_request_length-1][0] < time
+  end
+
+  def add_to_and_reorder_worst_requests(req_entry)
+    @worst_requests << req_entry
+    @worst_requests.sort! {|a,b| (b[0] && a[0]) ? b[0]<=>a[0] : 0}
+    @worst_requests = @worst_requests[0,@worst_request_length]
+  end
+
+  def print_stats
+    puts "Printing report for total request times"
+    puts "--------"
+    puts HEADER
+    puts @total_stat.to_s
+    puts "--------"
+    @stat_hash.print()
+    puts "\nTop #{@sorted_limit} by Count"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"count",:limit=>@sorted_limit,:ascending=>false)
+    puts "\nTop #{@sorted_limit} by Sum of Time"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"sum",:limit=>@sorted_limit,:ascending=>false)
+    puts "\nTop #{@sorted_limit} Greatest Max"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"max",:limit=>@sorted_limit,:ascending=>false)
+    puts "\nTop #{@sorted_limit} Least Min"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"min",:limit=>@sorted_limit)
+    puts "\nTop #{@sorted_limit} Greatest Median"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"median",:limit=>@sorted_limit,:ascending=>false)
+    puts "\nTop #{@sorted_limit} Greatest Standard Deviation"
+    puts HEADER
+    @stat_hash.print(:sort_by=>"standard_deviation",:limit=>@sorted_limit,:ascending=>false)
+    puts "\n#{@worst_request_length} Worst Requests"
+    @worst_requests.each {|w| puts w[1]}
+    puts "\nCompleted report in #{(Time.now-@start_time)/60.0} minutes"
+  end
+end

data/lib/stat.rb

@@ -0,0 +1,55 @@
+class Stat
+  def initialize(key)
+    @key=key
+    @min = nil
+    @max = nil
+    @sum = 0
+    @sum_squares = 0
+    @count = 0
+    @values = []
+  end
+  def add(value)
+    value=1.0*value
+    @count+=1
+    @min = value unless @min
+    @min = value if value<@min
+    @max = value unless @max
+    @max = value if value>@max
+    @sum += value
+    @sum_squares += value*value
+    @values << value
+  end
+  def key
+    @key
+  end
+  def count
+    @count
+  end
+  def sum
+    @sum
+  end
+  def min
+    @min
+  end
+  def max
+    @max
+  end
+  def average
+    @sum/@count
+  end
+  def median
+    return nil unless @values
+    l = @values.length
+    return nil unless l>0
+    @values.sort!
+    return (@values[l/2-1]+@values[l/2])/2 if l%2==0
+    @values[(l+1)/2-1]
+  end
+  def standard_deviation
+    return 0 if @count<=1
+    Math.sqrt((@sum_squares - (@sum*@sum/@count))/ (@count) )
+  end
+  def to_s
+      sprintf("%-45s %6d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f",key,count,sum,max,median,average,min,standard_deviation)
+  end
+end

data/lib/stat_hash.rb

@@ -0,0 +1,26 @@
+
+class StatHash
+  def initialize
+    @stats = Hash.new
+  end
+  def add(key,time)
+    stat = @stats[key] || (@stats[key] = Stat.new(key))
+    stat.add(time)
+  end
+  def print(args={:sort_by=>'key',:ascending=>true,:limit=>nil})
+    values = @stats.values
+    order = (args[:ascending] || args[:ascending].nil?) ? 1 : -1
+    values.sort! {|a,b|
+      as = a.send(args[:sort_by])
+      bs = b.send(args[:sort_by])
+      (as && bs) ? order*(as<=>bs) : 0
+    }
+    #values.sort! {|a,b| a.key<=>b.key}
+    limit = args[:limit]
+    for stat in values
+      break if limit && limit<=0
+      puts stat.to_s
+      limit-=1 if limit
+    end
+  end
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification 
+name: rawkx
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Peter Zimbelman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-14 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: RawkX is a gem to allow you to get Rawk like statistics on logs of any format easily.
+email: pzimbelman@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/line_parser.rb
+- lib/rawkx.rb
+- lib/stat.rb
+- lib/stat_hash.rb
+- README
+has_rdoc: true
+homepage: http://github.com/pzimbelman/rawkx
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: rawkx
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: "Extendable Rawk: a simple modification of Rawk to allow for easier extendability."
+test_files: []
+