beaver 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +13 -0
- data/README.rdoc +105 -0
- data/bin/beaver +39 -0
- data/lib/beaver.rb +12 -0
- data/lib/beaver/beaver.rb +94 -0
- data/lib/beaver/dam.rb +181 -0
- data/lib/beaver/parsers/rails.rb +83 -0
- data/lib/beaver/request.rb +150 -0
- data/lib/beaver/utils.rb +107 -0
- metadata +72 -0
data/LICENSE
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright 2011 Jordan Hollinger
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.rdoc
ADDED
@@ -0,0 +1,105 @@
|
|
1
|
+
== Beaver, chewing through logs to make something useful
|
2
|
+
|
3
|
+
Beaver is a light DSL for parsing Rails production logs back into usable data. It answers questions like:
|
4
|
+
|
5
|
+
* How many failed logins have there been today? Were they for the same user? From the same IP?
|
6
|
+
* How many 500, 404, etc. errors yesterday? On what pages?
|
7
|
+
* How many Widgets were created yesterday, and with what data?
|
8
|
+
* Did anyone submit a form with the words "kill them all"? Yikes.
|
9
|
+
|
10
|
+
Beaver is *not* intended as a replacement for something like Google Analytics, as it is trying to answer some differents kinds of questions.
|
11
|
+
Rails logs contain some great information, but I've never found a good tool for using them programatically (except for speed analyzers).
|
12
|
+
Hopefully Beaver can fill that niche for you. Personally I find it quite useful with Logwatch.
|
13
|
+
|
14
|
+
Read the full documentation at http://jordanhollinger.com/docs/beaver/. The Beaver::Dam and Beaver::Request classes
|
15
|
+
are probably what you're after.
|
16
|
+
|
17
|
+
== Installation
|
18
|
+
Beaver is still somewhat experimental, but it has proved stable and robust enough for a non-beta release.
|
19
|
+
|
20
|
+
[sudo] gem install beaver
|
21
|
+
|
22
|
+
== Use the command-line Beaver
|
23
|
+
Write your beaver file:
|
24
|
+
|
25
|
+
hit :failed_login, :method => :post, :path => '/login', :status => 401
|
26
|
+
|
27
|
+
hit :new_widgets, :path => '/widgets', :method => :post, :status => 302 do
|
28
|
+
puts "A Widget named #{params[:widget][:name]} was created!"
|
29
|
+
end
|
30
|
+
|
31
|
+
hit :help, :path => %r|^/help| do
|
32
|
+
skip! if path == '/help/page_i_want_to_ignore'
|
33
|
+
puts "#{ip} looked for help at #{path}"
|
34
|
+
end
|
35
|
+
|
36
|
+
hit :server_error, :status => (500..503)
|
37
|
+
|
38
|
+
dam :failed_login do
|
39
|
+
puts "Failed logins:"
|
40
|
+
hits.group_by { |h| h.params[:username] } do |user, fails|
|
41
|
+
puts " #{user}"
|
42
|
+
fails.each do |hit|
|
43
|
+
puts " from #{hit.ip} at #{hit.time.to_s}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
dam :server_errors do
|
49
|
+
puts "Server errors:"
|
50
|
+
hits.group_by(&:status) do |status, errors|
|
51
|
+
puts " There were #{errors.size} #{status} errors"
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
Run ''beaver'' from the command line, passing in your beaver file and some logs:
|
56
|
+
|
57
|
+
beaver my_beaver_file.rb /var/www/rails-app/log/production.log*
|
58
|
+
|
59
|
+
== Example use with Logwatch
|
60
|
+
This assumes 1) you're rotating your Rails logs daily and 2) you're running logwatch daily.
|
61
|
+
|
62
|
+
Check your beaver DSL file into your Rails app, maybe under /script.
|
63
|
+
|
64
|
+
Add a logwatch config file for your new service at /etc/logwatch/conf/services/your_app.conf:
|
65
|
+
|
66
|
+
Title = "Your Rails App"
|
67
|
+
LogFile = NONE
|
68
|
+
|
69
|
+
In /etc/logwatch/scripts/services/your_app:
|
70
|
+
|
71
|
+
beaver /var/www/your_app/script/beaver.rb --yesterday /var/www/your_app/log/production.log{,.1}
|
72
|
+
|
73
|
+
== Use your own Beaver
|
74
|
+
Use Beaver like a library:
|
75
|
+
|
76
|
+
require 'rubygems'
|
77
|
+
require 'beaver'
|
78
|
+
|
79
|
+
# Logs from the last day or so
|
80
|
+
logs = ['/var/www/rails-app/log/production.log',
|
81
|
+
'/var/www/rails-app/log/production.log.1']
|
82
|
+
|
83
|
+
# Parse the logs, but only include requests from yesterday
|
84
|
+
Beaver.parse logs, :on => Date.today-1 do
|
85
|
+
# Same DSL as above
|
86
|
+
hit :failed_login, :method => :post, :path => '/login', :status => 401
|
87
|
+
...
|
88
|
+
end
|
89
|
+
|
90
|
+
== Your Rails app should return appropriate HTTP statuses
|
91
|
+
Rails does a lot of great things for us, but one thing largely up to us are
|
92
|
+
HTTP status codes. For example, your failed logins are probably returning
|
93
|
+
200 when they should arguably be returning 400 or 401.
|
94
|
+
|
95
|
+
render :action => :login, :status => 401
|
96
|
+
|
97
|
+
See, it's easy. And very useful to Beaver.
|
98
|
+
|
99
|
+
== TODO
|
100
|
+
* Add support for Apache/Nginx/Rack::CommonLogger
|
101
|
+
|
102
|
+
== License
|
103
|
+
Copyright 2011 Jordan Hollinger
|
104
|
+
|
105
|
+
Licensed under the Apache License
|
data/bin/beaver
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'rubygems'
|
5
|
+
require 'beaver'
|
6
|
+
|
7
|
+
#
|
8
|
+
# This application was installed by RubyGems as part of the 'beaver' gem.
|
9
|
+
#
|
10
|
+
|
11
|
+
# Parse a string from the command-line into a Date object
|
12
|
+
def parse_date(date)
|
13
|
+
case date
|
14
|
+
when /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ then Date.parse(date)
|
15
|
+
when /^-\d+$/ then Date.today + date.to_i
|
16
|
+
else nil
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
options = {}
|
21
|
+
o = OptionParser.new do |opts|
|
22
|
+
opts.banner = 'Usage: beaver dams.rb [options] /path/to/rails/production.log [/another/log [/yet/another/log]]'
|
23
|
+
opts.on('--on DATE', 'Only include log entries from the given date (yyyy-mm-dd or -n days)') { |d| options[:on] = parse_date(d) }
|
24
|
+
opts.on('--after DATE', 'Only include log entries from after the given date (yyyy-mm-dd or -n days)') { |d| options[:after] = parse_date(d) }
|
25
|
+
opts.on('--before DATE', 'Only include log entries from before the given date (yyyy-mm-dd or -n days)') { |d| options[:before] = parse_date(d) }
|
26
|
+
opts.on('--today', 'Alias to --on=-0') { options[:on] = Date.today }
|
27
|
+
opts.on('--yesterday', 'Alias to --on=-1') { options[:on] = Date.today-1 }
|
28
|
+
end
|
29
|
+
o.parse!
|
30
|
+
|
31
|
+
if ARGV.size < 2
|
32
|
+
puts o.banner
|
33
|
+
exit 1
|
34
|
+
end
|
35
|
+
|
36
|
+
beaver_file = File.open(ARGV.shift, &:read)
|
37
|
+
Beaver.parse *[ARGV, options].flatten do
|
38
|
+
eval beaver_file
|
39
|
+
end
|
data/lib/beaver.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
begin
|
2
|
+
require 'zlib'
|
3
|
+
rescue LoadError
|
4
|
+
$stderr.puts "Zlib not available; compressed log files will be skipped."
|
5
|
+
end
|
6
|
+
|
7
|
+
require 'beaver/utils'
|
8
|
+
require 'beaver/beaver'
|
9
|
+
require 'beaver/dam'
|
10
|
+
require 'beaver/request'
|
11
|
+
|
12
|
+
require 'beaver/parsers/rails'
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# Not specifically a performance analyzer (like https://github.com/wvanbergen/request-log-analyzer/wiki)
|
2
|
+
# Rather, a DSL for finding out how people are using your Rails app (which could include performance).
|
3
|
+
module Beaver
|
4
|
+
# Alias to creating a new Beaver, parsing the files, and filtering them
|
5
|
+
def self.parse(*args, &blk)
|
6
|
+
raise ArgumentError, 'You must pass a block to Beaver#parse' unless block_given?
|
7
|
+
beaver = Beaver.new(*args)
|
8
|
+
beaver.parse
|
9
|
+
beaver.filter(&blk)
|
10
|
+
beaver
|
11
|
+
end
|
12
|
+
|
13
|
+
# Alias to creating a new Beaver
|
14
|
+
def self.new(*args)
|
15
|
+
Beaver.new(*args)
|
16
|
+
end
|
17
|
+
|
18
|
+
# The Beaver class, which keeps track of the files you're parsing, the Beaver::Dam objects you've defined,
|
19
|
+
# and parses and stores the matching Beaver::Request objects.
|
20
|
+
class Beaver
|
21
|
+
# The files to parse
|
22
|
+
attr_reader :files
|
23
|
+
# The Beaver::Dam objects you're defined
|
24
|
+
attr_reader :dams
|
25
|
+
# The Beaver::Request objects matched in the given files
|
26
|
+
attr_reader :requests
|
27
|
+
|
28
|
+
# Pass in globs or file paths. The final argument may be an options Hash.
|
29
|
+
# These options will be applied as matchers to all hits. See Beaver::Dam for available options.
|
30
|
+
def initialize(*args)
|
31
|
+
@global_matchers = args.last.is_a?(Hash) ? args.pop : {}
|
32
|
+
@files = args.map { |a| Dir.glob(a) }
|
33
|
+
@files.flatten!
|
34
|
+
@requests, @dams, @sums = [], {}, {}
|
35
|
+
end
|
36
|
+
|
37
|
+
# Creates a new Dam and appends it to this Beaver. name should be a unique symbol.
|
38
|
+
# See Beaver::Dam for available options.
|
39
|
+
def hit(dam_name, matchers={}, &callback)
|
40
|
+
raise ArgumentError, "A dam named #{dam_name} already exists" if @dams.has_key? dam_name
|
41
|
+
matchers = @global_matchers.merge matchers
|
42
|
+
@dams[dam_name] = Dam.new(dam_name, matchers, &callback)
|
43
|
+
end
|
44
|
+
|
45
|
+
# Define a sumarry for a Dam
|
46
|
+
def dam(name, &callback)
|
47
|
+
raise ArgumentError, "Unable to find a Dam named #{name}" unless @dams.has_key? name
|
48
|
+
@sums[name] = callback
|
49
|
+
end
|
50
|
+
|
51
|
+
# Parse the logs and filter them through the dams
|
52
|
+
# "parse" must be run before this, or there will be no requests
|
53
|
+
def filter(&blk)
|
54
|
+
instance_eval(&blk) if block_given?
|
55
|
+
@requests.each do |req|
|
56
|
+
@dams.each_value do |dam|
|
57
|
+
if dam.matches? req
|
58
|
+
catch :skip do
|
59
|
+
req.instance_eval(&dam.callback) if dam.callback
|
60
|
+
dam.hits << req
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
@sums.each do |dam_name, callback|
|
66
|
+
@dams[dam_name].instance_eval(&callback) if @dams[dam_name].hits.any?
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Parse the logs into @requests
|
71
|
+
def parse
|
72
|
+
@files.each do |file|
|
73
|
+
zipped = file =~ /\.gz\Z/i
|
74
|
+
next if zipped and not defined? Zlib
|
75
|
+
File.open(file, 'r:UTF-8') do |f|
|
76
|
+
handle = (zipped ? Zlib::GzipReader.new(f) : f)
|
77
|
+
request = nil
|
78
|
+
handle.each_line do |line|
|
79
|
+
request = Request.for(line).new if request.nil?
|
80
|
+
if request.bad?
|
81
|
+
request = nil
|
82
|
+
next
|
83
|
+
end
|
84
|
+
request << line
|
85
|
+
if request.completed?
|
86
|
+
@requests << request
|
87
|
+
request = nil
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
data/lib/beaver/dam.rb
ADDED
@@ -0,0 +1,181 @@
|
|
1
|
+
module Beaver
|
2
|
+
# A Dam "traps" certain Requests, using one or more matching options. A request must meet *all* of the
|
3
|
+
# matching options specified.
|
4
|
+
#
|
5
|
+
# Matchers:
|
6
|
+
#
|
7
|
+
# :path => String for exact match, or Regex
|
8
|
+
#
|
9
|
+
# :method => A Symbol of :get, :post, :update or :delete, or any array of any (reads the magic _method field if present)
|
10
|
+
#
|
11
|
+
# :status => A Fixnum like 404 or a Range like (500..503)
|
12
|
+
#
|
13
|
+
# :ip => String for exact match, or Regex
|
14
|
+
#
|
15
|
+
# :longer_than => Fixnum n. Matches any request which took longer than n milliseconds to complete.
|
16
|
+
#
|
17
|
+
# :shorter_than => Fixnum n. Matches any request which took less than n milliseconds to complete.
|
18
|
+
#
|
19
|
+
# :before => Date or Time for which the request must have ocurred before
|
20
|
+
#
|
21
|
+
# :after => Date or Time for which the request must have ocurred after
|
22
|
+
#
|
23
|
+
# :on => Date - the request must have ocurred on this date
|
24
|
+
#
|
25
|
+
# :params_str => A regular expressing matching the Parameters string
|
26
|
+
#
|
27
|
+
# :params => A Hash of Symbol=>String/Regexp pairs: {:username => 'bob', :email => /@gmail\.com$/}. All must match.
|
28
|
+
#
|
29
|
+
# :match => A "catch-all" Regex that will be matched against the entire request string
|
30
|
+
#
|
31
|
+
# The last argument may be a block, which will be called everytime this Dam is hit.
|
32
|
+
# The block will be run in the context of the Request object. This can be used for
|
33
|
+
# further checks or for reporting purposes.
|
34
|
+
class Dam
|
35
|
+
# The symbol name of this Beaver::Dam
|
36
|
+
attr_reader :name
|
37
|
+
# An optional callback when a Beaver::Request hits this Dam
|
38
|
+
attr_reader :callback
|
39
|
+
# An array of Beaver::Request objects that have hit this Dam
|
40
|
+
attr_reader :hits
|
41
|
+
|
42
|
+
# Name should be a unique symbol. Matchers is an options Hash. The callback will be evauluated within
|
43
|
+
# the context of a Beaver::Request.
|
44
|
+
def initialize(name, matchers, &callback)
|
45
|
+
@name = name
|
46
|
+
@callback = callback
|
47
|
+
@hits = []
|
48
|
+
set_matchers(matchers)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Returns an array of IP address that hit this Dam.
|
52
|
+
def ips
|
53
|
+
@ips ||= @hits.map(&:ip).uniq
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns true if the given Request hits this Dam, false if not.
|
57
|
+
def matches?(request)
|
58
|
+
return false if request.final?
|
59
|
+
return false unless @match_path_s.nil? or @match_path_s == request.path
|
60
|
+
return false unless @match_path_r.nil? or @match_path_r =~ request.path
|
61
|
+
return false unless @match_longer.nil? or @match_longer < request.ms
|
62
|
+
return false unless @match_shorter.nil? or @match_shorter > request.ms
|
63
|
+
return false unless @match_method_s.nil? or @match_method_s == request.method
|
64
|
+
return false unless @match_method_a.nil? or @match_method_a.include? request.method
|
65
|
+
return false unless @match_status.nil? or @match_status === request.status
|
66
|
+
return false unless @match_ip_s.nil? or @match_ip_s == request.ip
|
67
|
+
return false unless @match_ip_r.nil? or @match_ip_r =~ request.ip
|
68
|
+
return false unless @match_before.nil? or @match_before > request.time
|
69
|
+
return false unless @match_after.nil? or @match_after < request.time
|
70
|
+
return false unless @match_on.nil? or (@match_on.year == request.time.year and @match_on.month == request.time.month and @match_on.day == request.time.day)
|
71
|
+
return false unless @match_params_str.nil? or @match_params_str =~ request.params_str
|
72
|
+
return false unless @match_r.nil? or @match_r =~ request.to_s
|
73
|
+
return false unless @match_params.nil? or matching_hashes?(@match_params, request.params)
|
74
|
+
return true
|
75
|
+
end
|
76
|
+
|
77
|
+
private
|
78
|
+
|
79
|
+
# Recursively compares to Hashes. If all of Hash A is in Hash B, they match.
|
80
|
+
def matching_hashes?(a,b)
|
81
|
+
intersecting_keys = a.keys & b.keys
|
82
|
+
if intersecting_keys.any?
|
83
|
+
a_values = a.values_at(*intersecting_keys)
|
84
|
+
b_values = b.values_at(*intersecting_keys)
|
85
|
+
indicies = (0..b_values.size-1)
|
86
|
+
indicies.all? do |i|
|
87
|
+
if a_values[i].is_a? String
|
88
|
+
a_values[i] == b_values[i]
|
89
|
+
elsif a_values[i].is_a?(Regexp) and b_values[i].is_a?(String)
|
90
|
+
a_values[i] =~ b_values[i]
|
91
|
+
elsif a_values[i].is_a?(Hash) and b_values[i].is_a?(Hash)
|
92
|
+
matching_hashes? a_values[i], b_values[i]
|
93
|
+
else
|
94
|
+
false
|
95
|
+
end
|
96
|
+
end
|
97
|
+
else
|
98
|
+
false
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# Parses and checks the validity of the matching options passed to the Dam.
|
103
|
+
def set_matchers(matchers)
|
104
|
+
case matchers[:path].class.name
|
105
|
+
when String.name then @match_path_s = matchers[:path]
|
106
|
+
when Regexp.name then @match_path_r = matchers[:path]
|
107
|
+
else raise ArgumentError, "Path must be a String or a Regexp (it's a #{matchers[:path].class.name})"
|
108
|
+
end if matchers[:path]
|
109
|
+
|
110
|
+
case matchers[:method].class.name
|
111
|
+
when Symbol.name then @match_method_s = matchers[:method]
|
112
|
+
when Array.name then @match_method_a = matchers[:method]
|
113
|
+
else raise ArgumentError, "Method must be a Symbol or an Array (it's a #{matchers[:method].class.name})"
|
114
|
+
end if matchers[:method]
|
115
|
+
|
116
|
+
case matchers[:status].class.name
|
117
|
+
when Fixnum.name, Range.name then @match_status = matchers[:status]
|
118
|
+
else raise ArgumentError, "Status must be a Fixnum or a Range (it's a #{matchers[:status].class.name})"
|
119
|
+
end if matchers[:status]
|
120
|
+
|
121
|
+
case matchers[:ip].class.name
|
122
|
+
when String.name then @match_status_s = matchers[:ip]
|
123
|
+
when Regexp.name then @match_status_r = matchers[:ip]
|
124
|
+
else raise ArgumentError, "IP must be a String or a Regexp (it's a #{matchers[:ip].class.name})"
|
125
|
+
end if matchers[:ip]
|
126
|
+
|
127
|
+
case matchers[:longer_than].class.name
|
128
|
+
when Fixnum.name then @match_longer = matchers[:longer_than]
|
129
|
+
else raise ArgumentError, "longer_than must be a Fixnum (it's a #{matchers[:longer_than].class.name})"
|
130
|
+
end if matchers[:longer_than]
|
131
|
+
|
132
|
+
case matchers[:shorter_than].class.name
|
133
|
+
when Fixnum.name then @match_shorter = matchers[:shorter_than]
|
134
|
+
else raise ArgumentError, "shorter_than must be a Fixnum (it's a #{matchers[:shorter_than].class.name})"
|
135
|
+
end if matchers[:shorter_than]
|
136
|
+
|
137
|
+
if matchers[:before]
|
138
|
+
@match_before = if matchers[:before].is_a? Time
|
139
|
+
matchers[:before]
|
140
|
+
elsif matchers[:before].is_a? Date
|
141
|
+
Utils::NormalizedTime.new(matchers[:before].year, matchers[:before].month, matchers[:before].day)
|
142
|
+
else
|
143
|
+
raise ArgumentError, "before must be a Date or Time (it's a #{matchers[:before].class.name})"
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
if matchers[:after]
|
148
|
+
@match_after = if matchers[:after].is_a? Time
|
149
|
+
matchers[:after]
|
150
|
+
elsif matchers[:after].is_a? Date
|
151
|
+
Utils::NormalizedTime.new(matchers[:after].year, matchers[:after].month, matchers[:after].day)
|
152
|
+
else
|
153
|
+
raise ArgumentError, "after must be a Date or Time (it's a #{matchers[:after].class.name})"
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
if matchers[:on]
|
158
|
+
if matchers[:on].is_a? Date
|
159
|
+
@match_on = matchers[:on]
|
160
|
+
else
|
161
|
+
raise ArgumentError, "on must be a Date (it's a #{matchers[:on].class.name})"
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
case matchers[:params_str].class.name
|
166
|
+
when Regexp.name then @match_params_str = matchers[:params_str]
|
167
|
+
else raise ArgumentError, "Params String must be a Regexp (it's a #{matchers[:params_str].class.name})"
|
168
|
+
end if matchers[:params_str]
|
169
|
+
|
170
|
+
case matchers[:params].class.name
|
171
|
+
when Hash.name then @match_params = matchers[:params]
|
172
|
+
else raise ArgumentError, "Params must be a String or a Regexp (it's a #{matchers[:params].class.name})"
|
173
|
+
end if matchers[:params]
|
174
|
+
|
175
|
+
case matchers[:match].class.name
|
176
|
+
when Regexp.name then @match_r = matchers[:match]
|
177
|
+
else raise ArgumentError, "Match must be a Regexp (it's a #{matchers[:match].class.name})"
|
178
|
+
end if matchers[:match]
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Beaver
|
2
|
+
module Parsers
|
3
|
+
# This appears to work with Rails 3 logs
|
4
|
+
class Rails < Request
|
5
|
+
# Tell the Request class to use this parser to parse logs
|
6
|
+
Request << self
|
7
|
+
|
8
|
+
REGEX_METHOD = /^Started ([A-Z]+)/
|
9
|
+
REGEX_METHOD_OVERRIDE = /"_method"=>"([A-Z]+)"/i
|
10
|
+
REGEX_COMPLETED = /^Completed (\d+)/
|
11
|
+
REGEX_PATH = /^Started \w{3,4} "([^"]+)"/
|
12
|
+
REGEX_PARAMS_STR = /^ Parameters: (\{.+\})$/
|
13
|
+
REGEX_IP = /" for (\d+[\d.]+) at /
|
14
|
+
REGEX_MS = / in (\d+)ms/
|
15
|
+
# Depending on the version of Rails, the time format may be wildly different
|
16
|
+
REGEX_TIME = / at ([a-z0-9:\+\- ]+)$/i
|
17
|
+
|
18
|
+
# Returns true if the given lines look like a Rails request
|
19
|
+
def self.match?(lines)
|
20
|
+
REGEX_METHOD =~ lines
|
21
|
+
end
|
22
|
+
|
23
|
+
# Returns true, always
|
24
|
+
def valid?; true; end
|
25
|
+
|
26
|
+
# Returns true if/when we have the final line of the multi-line Rails request
|
27
|
+
def completed?
|
28
|
+
REGEX_COMPLETED =~ @lines
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
# Parses and returns the request path
|
34
|
+
def parse_path
|
35
|
+
m = REGEX_PATH.match(@lines)
|
36
|
+
m ? m.captures.first : BLANK_STR
|
37
|
+
end
|
38
|
+
|
39
|
+
# Parses and returns the request method
|
40
|
+
def parse_method
|
41
|
+
m = REGEX_METHOD_OVERRIDE.match(@lines)
|
42
|
+
m = REGEX_METHOD.match(@lines) if m.nil?
|
43
|
+
m ? m.captures.first.downcase.to_sym : :method
|
44
|
+
end
|
45
|
+
|
46
|
+
# Parses and returns the response status
|
47
|
+
def parse_status
|
48
|
+
m = REGEX_COMPLETED.match(@lines)
|
49
|
+
m ? m.captures.first.to_i : 0
|
50
|
+
end
|
51
|
+
|
52
|
+
# Parses and returns the request parameters as a String
|
53
|
+
def parse_params_str
|
54
|
+
m = REGEX_PARAMS_STR.match(@lines)
|
55
|
+
m ? m.captures.first : BLANK_STR
|
56
|
+
end
|
57
|
+
|
58
|
+
# Parses and returns the request parameters as a Hash (relatively expensive)
|
59
|
+
def parse_params
|
60
|
+
p = params_str
|
61
|
+
p.empty? ? BLANK_HASH : Utils.str_to_hash(p)
|
62
|
+
end
|
63
|
+
|
64
|
+
# Parses and returns the request's IP address
|
65
|
+
def parse_ip
|
66
|
+
m = REGEX_IP.match(@lines)
|
67
|
+
m ? m.captures.first : BLANK_STR
|
68
|
+
end
|
69
|
+
|
70
|
+
# Parses and returns the number of milliseconds it took for the request to complete
|
71
|
+
def parse_ms
|
72
|
+
m = REGEX_MS.match(@lines)
|
73
|
+
m ? m.captures.first.to_i : 0
|
74
|
+
end
|
75
|
+
|
76
|
+
# Parses and returns the time at which the request was made
|
77
|
+
def parse_time
|
78
|
+
m = REGEX_TIME.match(@lines)
|
79
|
+
m ? Time.parse(m.captures.first) : nil
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,150 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module Beaver
|
4
|
+
# Represents a single request from the logs.
|
5
|
+
class Request
|
6
|
+
BLANK_STR = ''
|
7
|
+
BLANK_HASH = {}
|
8
|
+
|
9
|
+
# Holds the Parser classes used to parser requests
|
10
|
+
@types = []
|
11
|
+
|
12
|
+
# Add a child Request parser.
|
13
|
+
def self.<<(type)
|
14
|
+
@types << type
|
15
|
+
end
|
16
|
+
|
17
|
+
# Returns the correct Request parser class for the given log lines. If one cannot be found,
|
18
|
+
# the BadRequest class is returned, which the caller will want to ignore.
|
19
|
+
def self.for(lines)
|
20
|
+
@types.select { |t| t.match? lines }.first || BadRequest
|
21
|
+
end
|
22
|
+
|
23
|
+
# Accepts a String of log lines, presumably ones which belong to a single request.
|
24
|
+
def initialize(lines=nil)
|
25
|
+
@lines = lines || ''
|
26
|
+
@final = false
|
27
|
+
end
|
28
|
+
|
29
|
+
# Returns the log lines that make up this Request.
|
30
|
+
def to_s; @lines; end
|
31
|
+
|
32
|
+
# Returns true if this is a "good" request.
|
33
|
+
def good?; true; end
|
34
|
+
# Returns true if this is a "bad" request.
|
35
|
+
def bad?; not good?; end
|
36
|
+
|
37
|
+
# Append a log line
|
38
|
+
def <<(line)
|
39
|
+
@lines << line << $/
|
40
|
+
end
|
41
|
+
|
42
|
+
# Returns the request path
|
43
|
+
def path
|
44
|
+
@path ||= parse_path
|
45
|
+
end
|
46
|
+
|
47
|
+
# Returns the request method
|
48
|
+
def method
|
49
|
+
@method ||= parse_method
|
50
|
+
end
|
51
|
+
|
52
|
+
# Returns the response status
|
53
|
+
def status
|
54
|
+
@status ||= parse_status
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns the request parameters as a String
|
58
|
+
def params_str
|
59
|
+
@params_str ||= parse_params_str
|
60
|
+
end
|
61
|
+
|
62
|
+
# Returns the request parameters as a Hash (this is more expensive than Request#params_str)
|
63
|
+
def params
|
64
|
+
@params ||= parse_params
|
65
|
+
end
|
66
|
+
|
67
|
+
# Returns the IP address of the request
|
68
|
+
def ip
|
69
|
+
@ip ||= parse_ip
|
70
|
+
end
|
71
|
+
|
72
|
+
# Returns the number of milliseconds it took for the request to complete
|
73
|
+
def ms
|
74
|
+
@ms ||= parse_ms
|
75
|
+
end
|
76
|
+
|
77
|
+
# Returns the time at which the request was made
|
78
|
+
def time
|
79
|
+
@time ||= parse_time
|
80
|
+
end
|
81
|
+
|
82
|
+
# When called inside of a Beaver::Dam#hit block, this Request will *not* be matched.
|
83
|
+
def skip!
|
84
|
+
throw :skip
|
85
|
+
end
|
86
|
+
|
87
|
+
# When called inside of a Beaver::Dam#hit block, this Request will not match against any other Beaver::Dam.
|
88
|
+
def final!
|
89
|
+
@final = true
|
90
|
+
end
|
91
|
+
|
92
|
+
# Returns true if this Request should not be matched against any more Dams.
|
93
|
+
def final?
|
94
|
+
@final
|
95
|
+
end
|
96
|
+
|
97
|
+
# Returns true if the request has all the information it needs to be properly parsed
|
98
|
+
def completed?
|
99
|
+
true
|
100
|
+
end
|
101
|
+
|
102
|
+
protected
|
103
|
+
|
104
|
+
# Parses and returns the request path
|
105
|
+
def parse_path
|
106
|
+
BLANK_STR
|
107
|
+
end
|
108
|
+
|
109
|
+
# Parses and returns the request method
|
110
|
+
def parse_method
|
111
|
+
:method
|
112
|
+
end
|
113
|
+
|
114
|
+
# Parses and returns the response status
|
115
|
+
def parse_status
|
116
|
+
0
|
117
|
+
end
|
118
|
+
|
119
|
+
# Parses and returns the request params as a String
|
120
|
+
def parse_params_str
|
121
|
+
BLANK_STR
|
122
|
+
end
|
123
|
+
|
124
|
+
# Parses and returns the request params as a Hash
|
125
|
+
def parse_params
|
126
|
+
BLANK_HASH
|
127
|
+
end
|
128
|
+
|
129
|
+
# Parses and returns the request IP address
|
130
|
+
def parse_ip
|
131
|
+
BLANK_STR
|
132
|
+
end
|
133
|
+
|
134
|
+
# Parses and returns the number of milliseconds it took for the request to complete
|
135
|
+
def parse_ms
|
136
|
+
0
|
137
|
+
end
|
138
|
+
|
139
|
+
# Parses and returns the time at which the request was made
|
140
|
+
def parse_time
|
141
|
+
nil
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
# Represents a BadRequest that no parser could figure out.
|
146
|
+
class BadRequest < Request
|
147
|
+
# Returns false, always
|
148
|
+
def good?; false; end
|
149
|
+
end
|
150
|
+
end
|
data/lib/beaver/utils.rb
ADDED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
module Beaver
|
5
|
+
# Sundry utility methods and classes for use by Beaver
|
6
|
+
module Utils
|
7
|
+
LBRACE, RBRACE = '{', '}'
|
8
|
+
LBRACKET, RBRACKET = '[', ']'
|
9
|
+
QUOTE = '"'
|
10
|
+
ESCAPE = '\\'
|
11
|
+
EQUAL = '='
|
12
|
+
COLIN = ':'
|
13
|
+
TO_SPACE = ['>']
|
14
|
+
SPACE = ' '
|
15
|
+
COMMA = ','
|
16
|
+
LETTER_REGEX = /^[a-z]$/i
|
17
|
+
|
18
|
+
# Converts a string representation of a Hash into YAML, then into a Hash.
|
19
|
+
# This is targeted towards the Parameters value in Rails logs. It is assumed that every key is a represented as a String in the logs.
|
20
|
+
# All keys, except for numeric keys, will be converted to Symbols.
|
21
|
+
def self.str_to_hash(str)
|
22
|
+
s = ''
|
23
|
+
indent = 0
|
24
|
+
state = :pre_key
|
25
|
+
i = 0
|
26
|
+
str.each_char do |c|
|
27
|
+
i += 1
|
28
|
+
case c
|
29
|
+
when QUOTE
|
30
|
+
case state
|
31
|
+
when :pre_key
|
32
|
+
s << (SPACE * indent)
|
33
|
+
s << COLIN if str[i,1] =~ LETTER_REGEX
|
34
|
+
state = :key
|
35
|
+
next
|
36
|
+
when :key
|
37
|
+
s << COLIN << SPACE
|
38
|
+
state = :pre_val
|
39
|
+
next
|
40
|
+
when :pre_val, :escape
|
41
|
+
state = :val
|
42
|
+
next
|
43
|
+
when :val
|
44
|
+
state = :pre_key
|
45
|
+
s << "\n"
|
46
|
+
next
|
47
|
+
end
|
48
|
+
when LBRACE
|
49
|
+
case state
|
50
|
+
# Hash as a value, starting a new indent level
|
51
|
+
when :pre_val
|
52
|
+
state = :pre_key
|
53
|
+
indent += 2
|
54
|
+
s << "\n"# << (SPACE * indent)
|
55
|
+
next
|
56
|
+
when :pre_key
|
57
|
+
next
|
58
|
+
end
|
59
|
+
when RBRACE
|
60
|
+
case state
|
61
|
+
when :pre_key
|
62
|
+
indent -= 2 if indent > 0
|
63
|
+
end
|
64
|
+
when LBRACKET
|
65
|
+
case state
|
66
|
+
when :pre_val
|
67
|
+
state = :val_array
|
68
|
+
end
|
69
|
+
when RBRACKET
|
70
|
+
case state
|
71
|
+
when :val_array
|
72
|
+
state = :val_array_end
|
73
|
+
end
|
74
|
+
when ESCAPE
|
75
|
+
if state == :val
|
76
|
+
state = :escape
|
77
|
+
next
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
case state
|
82
|
+
when :key, :val, :val_array
|
83
|
+
s << c
|
84
|
+
when :escape
|
85
|
+
s << c
|
86
|
+
state = :val
|
87
|
+
when :val_array_end
|
88
|
+
s << c << "\n"
|
89
|
+
state = :pre_key
|
90
|
+
end
|
91
|
+
end
|
92
|
+
YAML.load s
|
93
|
+
end
|
94
|
+
|
95
|
+
# Normalizes Time.new across Ruby 1.8 and 1.9.
|
96
|
+
# Accepts the same arguments as Time.
|
97
|
+
class NormalizedTime < ::Time
|
98
|
+
if RUBY_VERSION < '1.9'
|
99
|
+
# Returns a new NormalizedTime object.
|
100
|
+
def self.new(*args)
|
101
|
+
args.pop if args.last.is_a? String
|
102
|
+
local(*args)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
metadata
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: beaver
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 0
|
8
|
+
- 1
|
9
|
+
version: 0.0.1
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Jordan Hollinger
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2011-09-24 00:00:00 -04:00
|
18
|
+
default_executable:
|
19
|
+
dependencies: []
|
20
|
+
|
21
|
+
description: A simple DSL for helping you discover what people are doing with your Rails app
|
22
|
+
email: jordan@jordanhollinger.com
|
23
|
+
executables:
|
24
|
+
- beaver
|
25
|
+
extensions: []
|
26
|
+
|
27
|
+
extra_rdoc_files: []
|
28
|
+
|
29
|
+
files:
|
30
|
+
- lib/beaver.rb
|
31
|
+
- lib/beaver/dam.rb
|
32
|
+
- lib/beaver/beaver.rb
|
33
|
+
- lib/beaver/request.rb
|
34
|
+
- lib/beaver/parsers/rails.rb
|
35
|
+
- lib/beaver/utils.rb
|
36
|
+
- README.rdoc
|
37
|
+
- LICENSE
|
38
|
+
- bin/beaver
|
39
|
+
has_rdoc: true
|
40
|
+
homepage: http://github.com/jhollinger/beaver
|
41
|
+
licenses: []
|
42
|
+
|
43
|
+
post_install_message:
|
44
|
+
rdoc_options: []
|
45
|
+
|
46
|
+
require_paths:
|
47
|
+
- lib
|
48
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ">="
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
segments:
|
54
|
+
- 0
|
55
|
+
version: "0"
|
56
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
segments:
|
62
|
+
- 0
|
63
|
+
version: "0"
|
64
|
+
requirements: []
|
65
|
+
|
66
|
+
rubyforge_project:
|
67
|
+
rubygems_version: 1.3.7
|
68
|
+
signing_key:
|
69
|
+
specification_version: 3
|
70
|
+
summary: Rails production log parser
|
71
|
+
test_files: []
|
72
|
+
|