rawkx 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. data/README +43 -0
  2. data/lib/line_parser.rb +27 -0
  3. data/lib/rawkx.rb +110 -0
  4. data/lib/stat.rb +55 -0
  5. data/lib/stat_hash.rb +26 -0
  6. metadata +68 -0
data/README ADDED
@@ -0,0 +1,43 @@
1
+ RawkX - An Extendable Version of the Rawk Log Analyzer
2
+ Author: Peter Zimbelman
3
+ Git Repository: http://github.com/pzimbelman/rawkx
4
+
5
+ This project is based off the Rawk project (http://rubyforge.org/projects/rawk-the-logs/) and uses much of its original code.
6
+ RawkX is mostly a reorganization of this functionality to make extending the analyzer with different log formats easier.
7
+
8
+ This gem is designed to allow users to easily extend the Rawk analyzer to have it use a seperate method for parsing.
9
+ This will enable the analyzer to parse and perform calculations on logs of any format, not just Rails logs.
10
+
11
+ This is done by simply overriding the 'parse_line' method used by RawkX to obtain a key/value pair for tracking times (values) associated with
12
+ their action/request (keys). Upon doing this, you can call RawkX as you normally would and can get all the useful statistics you would normally
13
+ get against any Rails log.
14
+
15
+ Also note: by default (no override of parse_line), RawkX has the ability to parse Rails logs as Rawk always has.
16
+
17
+ Below is a simple example to illustrate using RawkX:
18
+
19
+ For instance, lets say your Log looked like this:
20
+ Format: time_to_complete action username
21
+ -------------------------------------------------
22
+ 3.24 Read user1
23
+ 5.0 Write user1
24
+ 2.245 Read user2
25
+ 3.4 Read user3
26
+ --------------------------------------------------
27
+
28
+ You could create a simple ruby file which contains the following:
29
+ --------------------------------------------------
30
+ require "rubygems"
31
+ require "rawkx"
32
+
33
+ class RawkX
34
+ def parse_line(line)
35
+ fields = line.split
36
+ return fields[1], fields[0]
37
+ end
38
+ end
39
+ RawkX.new
40
+ --------------------------------------------------
41
+
42
+ this file can be run with the command: ruby <ruby_file_name> -f <log_file_name>
43
+ Running this will produce Rawk results for your logs. you would be using the Actions (Read or Write) as your keys and the time_to_complete as your values
@@ -0,0 +1,27 @@
1
+
2
+ module LineParser
3
+ #Default parse implementation. This is the original Rawk implementation for a Rails log.
4
+ def parse_line(line)
5
+ @last_actions ||= {}
6
+ pid_regexp = /\(pid\:\d+\)/
7
+ if line.index("Processing ")==0
8
+ action = line.split[1]
9
+ pid = line[pid_regexp]
10
+ @last_actions[pid]=action if pid
11
+ return
12
+ end
13
+ return unless line.index("Completed in")==0
14
+ #get the pid unless we are forcing url tracking
15
+ pid = line[pid_regexp]
16
+ key = pid ? @last_actions[pid] : nil
17
+
18
+ time_string = line[/Completed in \d+\.\d+/]
19
+ time_string = time_string[/\d+\.\d+/] if time_string
20
+ time = time_string ? time_string.to_f : 0.0
21
+
22
+ #if pids are not specified then we use the url for hashing
23
+ #the below regexp turns "[http://spongecell.com/calendar/view/bob]" to "/calendar/view"
24
+ key = (line[/\[\S+\]/].gsub(/\S+\/\/(\w|\.)*/,''))[/\/\w*\/?\w*/] unless key
25
+ return key, time
26
+ end
27
+ end
data/lib/rawkx.rb ADDED
@@ -0,0 +1,110 @@
1
+ require "#{File.expand_path(File.dirname(__FILE__))}/stat"
2
+ require "#{File.expand_path(File.dirname(__FILE__))}/stat_hash"
3
+ require "#{File.expand_path(File.dirname(__FILE__))}/line_parser"
4
+
5
+
6
+ class RawkX
7
+ include LineParser
8
+
9
+ VERSION = "0.1.0"
10
+ HEADER = "Request Count Sum Max Median Avg Min Std"
11
+ HELP = "\nRawkX - Extendedable Rawk v#{VERSION}\n"+
12
+ "Created by Peter Zimbelman\n"+
13
+ "The foundation of this gem is based off of the Rawk project. This is mostly a refactoring of that gem to make extending it easier\n" +
14
+ "The log file is read from standard input unless the -f flag is specified.\n\n"+
15
+ "The options are as follows:\n\n"+
16
+ " -f <filename> Use the specified file instead of standard input.\n\n"+
17
+ " -h Display this help.\n\n"+
18
+ " -s <count> Display <count> results in each group of data.\n\n"+
19
+ " -w <count> Display the top <count> worst requests.\n\n"
20
+
21
+ def initialize
22
+ @start_time = Time.now
23
+ build_arg_hash
24
+ if @arg_hash.keys.include?("h")
25
+ puts HELP
26
+ else
27
+ init_args
28
+ build_stats
29
+ print_stats
30
+ end
31
+ end
32
+
33
+
34
+ private
35
+ def build_arg_hash
36
+ @arg_hash = Hash.new
37
+ last_key=nil
38
+ for a in $*
39
+ if a.index("-")==0 && a.length>1
40
+ a[1,1000].scan(/[a-z]|\?/).each {|c| @arg_hash[last_key=c]=nil}
41
+ @arg_hash[last_key] = a[/\d+/] if last_key
42
+ elsif a.index("-")!=0 && last_key
43
+ @arg_hash[last_key] = a
44
+ end
45
+ end
46
+ end
47
+
48
+ def init_args
49
+ @input = $stdin
50
+ @input = File.new(@arg_hash["f"]) if @arg_hash["f"]
51
+ @worst_request_length= (@arg_hash["w"] || 20).to_i
52
+ @sorted_limit = (@arg_hash["s"] || 20).to_i
53
+ @stat_hash = StatHash.new
54
+ @total_stat = Stat.new("All Requests")
55
+ @worst_requests = []
56
+ end
57
+
58
+ def build_stats
59
+ while line = @input.gets
60
+ key, time = parse_line(line)
61
+ next unless key
62
+ time = time.to_f unless time.is_a? Float
63
+ @stat_hash.add(key,time)
64
+ @total_stat.add(time)
65
+ if is_a_worst_request?(time)
66
+ add_to_and_reorder_worst_requests([time,line])
67
+ end
68
+ end
69
+ end
70
+
71
+ def is_a_worst_request?(time)
72
+ @worst_requests.length < @worst_request_length || @worst_requests[@worst_request_length-1][0] < time
73
+ end
74
+
75
+ def add_to_and_reorder_worst_requests(req_entry)
76
+ @worst_requests << req_entry
77
+ @worst_requests.sort! {|a,b| (b[0] && a[0]) ? b[0]<=>a[0] : 0}
78
+ @worst_requests = @worst_requests[0,@worst_request_length]
79
+ end
80
+
81
+ def print_stats
82
+ puts "Printing report for total request times"
83
+ puts "--------"
84
+ puts HEADER
85
+ puts @total_stat.to_s
86
+ puts "--------"
87
+ @stat_hash.print()
88
+ puts "\nTop #{@sorted_limit} by Count"
89
+ puts HEADER
90
+ @stat_hash.print(:sort_by=>"count",:limit=>@sorted_limit,:ascending=>false)
91
+ puts "\nTop #{@sorted_limit} by Sum of Time"
92
+ puts HEADER
93
+ @stat_hash.print(:sort_by=>"sum",:limit=>@sorted_limit,:ascending=>false)
94
+ puts "\nTop #{@sorted_limit} Greatest Max"
95
+ puts HEADER
96
+ @stat_hash.print(:sort_by=>"max",:limit=>@sorted_limit,:ascending=>false)
97
+ puts "\nTop #{@sorted_limit} Least Min"
98
+ puts HEADER
99
+ @stat_hash.print(:sort_by=>"min",:limit=>@sorted_limit)
100
+ puts "\nTop #{@sorted_limit} Greatest Median"
101
+ puts HEADER
102
+ @stat_hash.print(:sort_by=>"median",:limit=>@sorted_limit,:ascending=>false)
103
+ puts "\nTop #{@sorted_limit} Greatest Standard Deviation"
104
+ puts HEADER
105
+ @stat_hash.print(:sort_by=>"standard_deviation",:limit=>@sorted_limit,:ascending=>false)
106
+ puts "\n#{@worst_request_length} Worst Requests"
107
+ @worst_requests.each {|w| puts w[1]}
108
+ puts "\nCompleted report in #{(Time.now-@start_time)/60.0} minutes"
109
+ end
110
+ end
data/lib/stat.rb ADDED
@@ -0,0 +1,55 @@
1
+ class Stat
2
+ def initialize(key)
3
+ @key=key
4
+ @min = nil
5
+ @max = nil
6
+ @sum = 0
7
+ @sum_squares = 0
8
+ @count = 0
9
+ @values = []
10
+ end
11
+ def add(value)
12
+ value=1.0*value
13
+ @count+=1
14
+ @min = value unless @min
15
+ @min = value if value<@min
16
+ @max = value unless @max
17
+ @max = value if value>@max
18
+ @sum += value
19
+ @sum_squares += value*value
20
+ @values << value
21
+ end
22
+ def key
23
+ @key
24
+ end
25
+ def count
26
+ @count
27
+ end
28
+ def sum
29
+ @sum
30
+ end
31
+ def min
32
+ @min
33
+ end
34
+ def max
35
+ @max
36
+ end
37
+ def average
38
+ @sum/@count
39
+ end
40
+ def median
41
+ return nil unless @values
42
+ l = @values.length
43
+ return nil unless l>0
44
+ @values.sort!
45
+ return (@values[l/2-1]+@values[l/2])/2 if l%2==0
46
+ @values[(l+1)/2-1]
47
+ end
48
+ def standard_deviation
49
+ return 0 if @count<=1
50
+ Math.sqrt((@sum_squares - (@sum*@sum/@count))/ (@count) )
51
+ end
52
+ def to_s
53
+ sprintf("%-45s %6d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f",key,count,sum,max,median,average,min,standard_deviation)
54
+ end
55
+ end
data/lib/stat_hash.rb ADDED
@@ -0,0 +1,26 @@
1
+
2
+ class StatHash
3
+ def initialize
4
+ @stats = Hash.new
5
+ end
6
+ def add(key,time)
7
+ stat = @stats[key] || (@stats[key] = Stat.new(key))
8
+ stat.add(time)
9
+ end
10
+ def print(args={:sort_by=>'key',:ascending=>true,:limit=>nil})
11
+ values = @stats.values
12
+ order = (args[:ascending] || args[:ascending].nil?) ? 1 : -1
13
+ values.sort! {|a,b|
14
+ as = a.send(args[:sort_by])
15
+ bs = b.send(args[:sort_by])
16
+ (as && bs) ? order*(as<=>bs) : 0
17
+ }
18
+ #values.sort! {|a,b| a.key<=>b.key}
19
+ limit = args[:limit]
20
+ for stat in values
21
+ break if limit && limit<=0
22
+ puts stat.to_s
23
+ limit-=1 if limit
24
+ end
25
+ end
26
+ end
metadata ADDED
@@ -0,0 +1,68 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rawkx
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 1
8
+ - 0
9
+ version: 0.1.0
10
+ platform: ruby
11
+ authors:
12
+ - Peter Zimbelman
13
+ autorequire:
14
+ bindir: bin
15
+ cert_chain: []
16
+
17
+ date: 2010-04-14 00:00:00 -04:00
18
+ default_executable:
19
+ dependencies: []
20
+
21
+ description: RawkX is a gem to allow you to get Rawk like statistics on logs of any format easily.
22
+ email: pzimbelman@gmail.com
23
+ executables: []
24
+
25
+ extensions: []
26
+
27
+ extra_rdoc_files: []
28
+
29
+ files:
30
+ - lib/line_parser.rb
31
+ - lib/rawkx.rb
32
+ - lib/stat.rb
33
+ - lib/stat_hash.rb
34
+ - README
35
+ has_rdoc: true
36
+ homepage: http://github.com/pzimbelman/rawkx
37
+ licenses: []
38
+
39
+ post_install_message:
40
+ rdoc_options: []
41
+
42
+ require_paths:
43
+ - lib
44
+ required_ruby_version: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ segments:
49
+ - 0
50
+ version: "0"
51
+ required_rubygems_version: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ segments:
56
+ - 1
57
+ - 3
58
+ - 6
59
+ version: 1.3.6
60
+ requirements: []
61
+
62
+ rubyforge_project: rawkx
63
+ rubygems_version: 1.3.6
64
+ signing_key:
65
+ specification_version: 3
66
+ summary: "Extendable Rawk: a simple modification of Rawk to allow for easier extendability."
67
+ test_files: []
68
+