rawkx 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README +43 -0
- data/lib/line_parser.rb +27 -0
- data/lib/rawkx.rb +110 -0
- data/lib/stat.rb +55 -0
- data/lib/stat_hash.rb +26 -0
- metadata +68 -0
data/README
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
RawkX - An Extendable Version of the Rawk Log Analyzer
|
2
|
+
Author: Peter Zimbelman
|
3
|
+
Git Repository: http://github.com/pzimbelman/rawkx
|
4
|
+
|
5
|
+
This project is based off the Rawk project (http://rubyforge.org/projects/rawk-the-logs/) and uses much of its original code.
|
6
|
+
RawkX is mostly a reorganization of this functionality to make extending the analyzer with different log formats easier.
|
7
|
+
|
8
|
+
This gem is designed to allow users to easily extend the Rawk analyzer to have it use a seperate method for parsing.
|
9
|
+
This will enable the analyzer to parse and perform calculations on logs of any format, not just Rails logs.
|
10
|
+
|
11
|
+
This is done by simply overriding the 'parse_line' method used by RawkX to obtain a key/value pair for tracking times (values) associated with
|
12
|
+
their action/request (keys). Upon doing this, you can call RawkX as you normally would and can get all the useful statistics you would normally
|
13
|
+
get against any Rails log.
|
14
|
+
|
15
|
+
Also note: by default (no override of parse_line), RawkX has the ability to parse Rails logs as Rawk always has.
|
16
|
+
|
17
|
+
Below is a simple example to illustrate using RawkX:
|
18
|
+
|
19
|
+
For instance, lets say your Log looked like this:
|
20
|
+
Format: time_to_complete action username
|
21
|
+
-------------------------------------------------
|
22
|
+
3.24 Read user1
|
23
|
+
5.0 Write user1
|
24
|
+
2.245 Read user2
|
25
|
+
3.4 Read user3
|
26
|
+
--------------------------------------------------
|
27
|
+
|
28
|
+
You could create a simple ruby file which contains the following:
|
29
|
+
--------------------------------------------------
|
30
|
+
require "rubygems"
|
31
|
+
require "rawkx"
|
32
|
+
|
33
|
+
class RawkX
|
34
|
+
def parse_line(line)
|
35
|
+
fields = line.split
|
36
|
+
return fields[1], fields[0]
|
37
|
+
end
|
38
|
+
end
|
39
|
+
RawkX.new
|
40
|
+
--------------------------------------------------
|
41
|
+
|
42
|
+
this file can be run with the command: ruby <ruby_file_name> -f <log_file_name>
|
43
|
+
Running this will produce Rawk results for your logs. you would be using the Actions (Read or Write) as your keys and the time_to_complete as your values
|
data/lib/line_parser.rb
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
|
2
|
+
module LineParser
|
3
|
+
#Default parse implementation. This is the original Rawk implementation for a Rails log.
|
4
|
+
def parse_line(line)
|
5
|
+
@last_actions ||= {}
|
6
|
+
pid_regexp = /\(pid\:\d+\)/
|
7
|
+
if line.index("Processing ")==0
|
8
|
+
action = line.split[1]
|
9
|
+
pid = line[pid_regexp]
|
10
|
+
@last_actions[pid]=action if pid
|
11
|
+
return
|
12
|
+
end
|
13
|
+
return unless line.index("Completed in")==0
|
14
|
+
#get the pid unless we are forcing url tracking
|
15
|
+
pid = line[pid_regexp]
|
16
|
+
key = pid ? @last_actions[pid] : nil
|
17
|
+
|
18
|
+
time_string = line[/Completed in \d+\.\d+/]
|
19
|
+
time_string = time_string[/\d+\.\d+/] if time_string
|
20
|
+
time = time_string ? time_string.to_f : 0.0
|
21
|
+
|
22
|
+
#if pids are not specified then we use the url for hashing
|
23
|
+
#the below regexp turns "[http://spongecell.com/calendar/view/bob]" to "/calendar/view"
|
24
|
+
key = (line[/\[\S+\]/].gsub(/\S+\/\/(\w|\.)*/,''))[/\/\w*\/?\w*/] unless key
|
25
|
+
return key, time
|
26
|
+
end
|
27
|
+
end
|
data/lib/rawkx.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
require "#{File.expand_path(File.dirname(__FILE__))}/stat"
|
2
|
+
require "#{File.expand_path(File.dirname(__FILE__))}/stat_hash"
|
3
|
+
require "#{File.expand_path(File.dirname(__FILE__))}/line_parser"
|
4
|
+
|
5
|
+
|
6
|
+
class RawkX
|
7
|
+
include LineParser
|
8
|
+
|
9
|
+
VERSION = "0.1.0"
|
10
|
+
HEADER = "Request Count Sum Max Median Avg Min Std"
|
11
|
+
HELP = "\nRawkX - Extendedable Rawk v#{VERSION}\n"+
|
12
|
+
"Created by Peter Zimbelman\n"+
|
13
|
+
"The foundation of this gem is based off of the Rawk project. This is mostly a refactoring of that gem to make extending it easier\n" +
|
14
|
+
"The log file is read from standard input unless the -f flag is specified.\n\n"+
|
15
|
+
"The options are as follows:\n\n"+
|
16
|
+
" -f <filename> Use the specified file instead of standard input.\n\n"+
|
17
|
+
" -h Display this help.\n\n"+
|
18
|
+
" -s <count> Display <count> results in each group of data.\n\n"+
|
19
|
+
" -w <count> Display the top <count> worst requests.\n\n"
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@start_time = Time.now
|
23
|
+
build_arg_hash
|
24
|
+
if @arg_hash.keys.include?("h")
|
25
|
+
puts HELP
|
26
|
+
else
|
27
|
+
init_args
|
28
|
+
build_stats
|
29
|
+
print_stats
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
|
34
|
+
private
|
35
|
+
def build_arg_hash
|
36
|
+
@arg_hash = Hash.new
|
37
|
+
last_key=nil
|
38
|
+
for a in $*
|
39
|
+
if a.index("-")==0 && a.length>1
|
40
|
+
a[1,1000].scan(/[a-z]|\?/).each {|c| @arg_hash[last_key=c]=nil}
|
41
|
+
@arg_hash[last_key] = a[/\d+/] if last_key
|
42
|
+
elsif a.index("-")!=0 && last_key
|
43
|
+
@arg_hash[last_key] = a
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def init_args
|
49
|
+
@input = $stdin
|
50
|
+
@input = File.new(@arg_hash["f"]) if @arg_hash["f"]
|
51
|
+
@worst_request_length= (@arg_hash["w"] || 20).to_i
|
52
|
+
@sorted_limit = (@arg_hash["s"] || 20).to_i
|
53
|
+
@stat_hash = StatHash.new
|
54
|
+
@total_stat = Stat.new("All Requests")
|
55
|
+
@worst_requests = []
|
56
|
+
end
|
57
|
+
|
58
|
+
def build_stats
|
59
|
+
while line = @input.gets
|
60
|
+
key, time = parse_line(line)
|
61
|
+
next unless key
|
62
|
+
time = time.to_f unless time.is_a? Float
|
63
|
+
@stat_hash.add(key,time)
|
64
|
+
@total_stat.add(time)
|
65
|
+
if is_a_worst_request?(time)
|
66
|
+
add_to_and_reorder_worst_requests([time,line])
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def is_a_worst_request?(time)
|
72
|
+
@worst_requests.length < @worst_request_length || @worst_requests[@worst_request_length-1][0] < time
|
73
|
+
end
|
74
|
+
|
75
|
+
def add_to_and_reorder_worst_requests(req_entry)
|
76
|
+
@worst_requests << req_entry
|
77
|
+
@worst_requests.sort! {|a,b| (b[0] && a[0]) ? b[0]<=>a[0] : 0}
|
78
|
+
@worst_requests = @worst_requests[0,@worst_request_length]
|
79
|
+
end
|
80
|
+
|
81
|
+
def print_stats
|
82
|
+
puts "Printing report for total request times"
|
83
|
+
puts "--------"
|
84
|
+
puts HEADER
|
85
|
+
puts @total_stat.to_s
|
86
|
+
puts "--------"
|
87
|
+
@stat_hash.print()
|
88
|
+
puts "\nTop #{@sorted_limit} by Count"
|
89
|
+
puts HEADER
|
90
|
+
@stat_hash.print(:sort_by=>"count",:limit=>@sorted_limit,:ascending=>false)
|
91
|
+
puts "\nTop #{@sorted_limit} by Sum of Time"
|
92
|
+
puts HEADER
|
93
|
+
@stat_hash.print(:sort_by=>"sum",:limit=>@sorted_limit,:ascending=>false)
|
94
|
+
puts "\nTop #{@sorted_limit} Greatest Max"
|
95
|
+
puts HEADER
|
96
|
+
@stat_hash.print(:sort_by=>"max",:limit=>@sorted_limit,:ascending=>false)
|
97
|
+
puts "\nTop #{@sorted_limit} Least Min"
|
98
|
+
puts HEADER
|
99
|
+
@stat_hash.print(:sort_by=>"min",:limit=>@sorted_limit)
|
100
|
+
puts "\nTop #{@sorted_limit} Greatest Median"
|
101
|
+
puts HEADER
|
102
|
+
@stat_hash.print(:sort_by=>"median",:limit=>@sorted_limit,:ascending=>false)
|
103
|
+
puts "\nTop #{@sorted_limit} Greatest Standard Deviation"
|
104
|
+
puts HEADER
|
105
|
+
@stat_hash.print(:sort_by=>"standard_deviation",:limit=>@sorted_limit,:ascending=>false)
|
106
|
+
puts "\n#{@worst_request_length} Worst Requests"
|
107
|
+
@worst_requests.each {|w| puts w[1]}
|
108
|
+
puts "\nCompleted report in #{(Time.now-@start_time)/60.0} minutes"
|
109
|
+
end
|
110
|
+
end
|
data/lib/stat.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
class Stat
|
2
|
+
def initialize(key)
|
3
|
+
@key=key
|
4
|
+
@min = nil
|
5
|
+
@max = nil
|
6
|
+
@sum = 0
|
7
|
+
@sum_squares = 0
|
8
|
+
@count = 0
|
9
|
+
@values = []
|
10
|
+
end
|
11
|
+
def add(value)
|
12
|
+
value=1.0*value
|
13
|
+
@count+=1
|
14
|
+
@min = value unless @min
|
15
|
+
@min = value if value<@min
|
16
|
+
@max = value unless @max
|
17
|
+
@max = value if value>@max
|
18
|
+
@sum += value
|
19
|
+
@sum_squares += value*value
|
20
|
+
@values << value
|
21
|
+
end
|
22
|
+
def key
|
23
|
+
@key
|
24
|
+
end
|
25
|
+
def count
|
26
|
+
@count
|
27
|
+
end
|
28
|
+
def sum
|
29
|
+
@sum
|
30
|
+
end
|
31
|
+
def min
|
32
|
+
@min
|
33
|
+
end
|
34
|
+
def max
|
35
|
+
@max
|
36
|
+
end
|
37
|
+
def average
|
38
|
+
@sum/@count
|
39
|
+
end
|
40
|
+
def median
|
41
|
+
return nil unless @values
|
42
|
+
l = @values.length
|
43
|
+
return nil unless l>0
|
44
|
+
@values.sort!
|
45
|
+
return (@values[l/2-1]+@values[l/2])/2 if l%2==0
|
46
|
+
@values[(l+1)/2-1]
|
47
|
+
end
|
48
|
+
def standard_deviation
|
49
|
+
return 0 if @count<=1
|
50
|
+
Math.sqrt((@sum_squares - (@sum*@sum/@count))/ (@count) )
|
51
|
+
end
|
52
|
+
def to_s
|
53
|
+
sprintf("%-45s %6d %7.2f %7.2f %7.2f %7.2f %7.2f %7.2f",key,count,sum,max,median,average,min,standard_deviation)
|
54
|
+
end
|
55
|
+
end
|
data/lib/stat_hash.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
class StatHash
|
3
|
+
def initialize
|
4
|
+
@stats = Hash.new
|
5
|
+
end
|
6
|
+
def add(key,time)
|
7
|
+
stat = @stats[key] || (@stats[key] = Stat.new(key))
|
8
|
+
stat.add(time)
|
9
|
+
end
|
10
|
+
def print(args={:sort_by=>'key',:ascending=>true,:limit=>nil})
|
11
|
+
values = @stats.values
|
12
|
+
order = (args[:ascending] || args[:ascending].nil?) ? 1 : -1
|
13
|
+
values.sort! {|a,b|
|
14
|
+
as = a.send(args[:sort_by])
|
15
|
+
bs = b.send(args[:sort_by])
|
16
|
+
(as && bs) ? order*(as<=>bs) : 0
|
17
|
+
}
|
18
|
+
#values.sort! {|a,b| a.key<=>b.key}
|
19
|
+
limit = args[:limit]
|
20
|
+
for stat in values
|
21
|
+
break if limit && limit<=0
|
22
|
+
puts stat.to_s
|
23
|
+
limit-=1 if limit
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rawkx
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Peter Zimbelman
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-04-14 00:00:00 -04:00
|
18
|
+
default_executable:
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: RawkX is a gem to allow you to get Rawk like statistics on logs of any format easily.
|
22
|
+
email: pzimbelman@gmail.com
|
23
|
+
executables: []
|
24
|
+
|
25
|
+
extensions: []
|
26
|
+
|
27
|
+
extra_rdoc_files: []
|
28
|
+
|
29
|
+
files:
|
30
|
+
- lib/line_parser.rb
|
31
|
+
- lib/rawkx.rb
|
32
|
+
- lib/stat.rb
|
33
|
+
- lib/stat_hash.rb
|
34
|
+
- README
|
35
|
+
has_rdoc: true
|
36
|
+
homepage: http://github.com/pzimbelman/rawkx
|
37
|
+
licenses: []
|
38
|
+
|
39
|
+
post_install_message:
|
40
|
+
rdoc_options: []
|
41
|
+
|
42
|
+
require_paths:
|
43
|
+
- lib
|
44
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
segments:
|
49
|
+
- 0
|
50
|
+
version: "0"
|
51
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 1
|
57
|
+
- 3
|
58
|
+
- 6
|
59
|
+
version: 1.3.6
|
60
|
+
requirements: []
|
61
|
+
|
62
|
+
rubyforge_project: rawkx
|
63
|
+
rubygems_version: 1.3.6
|
64
|
+
signing_key:
|
65
|
+
specification_version: 3
|
66
|
+
summary: "Extendable Rawk: a simple modification of Rawk to allow for easier extendability."
|
67
|
+
test_files: []
|
68
|
+
|