sliding-stats 0.2.8
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +0 -0
- data/Rakefile +94 -0
- data/example/test.rb +60 -0
- data/features/stats.feature +34 -0
- data/features/step_definitions/stats_steps.rb +40 -0
- data/features/step_definitions/window_steps.rb +24 -0
- data/features/window.feature +19 -0
- data/lib/sliding-stats.rb +6 -0
- data/lib/sliding-stats/controller.rb +38 -0
- data/lib/sliding-stats/persist.rb +33 -0
- data/lib/sliding-stats/stats.rb +72 -0
- data/lib/sliding-stats/view.rb +90 -0
- data/lib/sliding-stats/window.rb +81 -0
- data/sliding-stats.gemspec +47 -0
- metadata +113 -0
data/README.rdoc
ADDED
File without changes
|
data/Rakefile
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
# Rakefile for SlidingStats. -*-ruby-*-
|
2
|
+
# Shamelessly stolen from Rack::Contrib
|
3
|
+
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
desc "Run all the tests"
|
8
|
+
#task :default => [:test]
|
9
|
+
|
10
|
+
#desc "Generate RDox"
|
11
|
+
#task "RDOX" do
|
12
|
+
# sh "specrb -Ilib:test -a --rdox >RDOX"
|
13
|
+
#end
|
14
|
+
|
15
|
+
#desc "Run all the fast tests"
|
16
|
+
#task :test do
|
17
|
+
# sh "specrb -Ilib:test -w #{ENV['TEST'] || '-a'} #{ENV['TESTOPTS']}"
|
18
|
+
#end
|
19
|
+
|
20
|
+
#desc "Run all the tests"
|
21
|
+
#task :fulltest do
|
22
|
+
# sh "specrb -Ilib:test -w #{ENV['TEST'] || '-a'} #{ENV['TESTOPTS']}"
|
23
|
+
#end
|
24
|
+
|
25
|
+
desc "Generate RDoc documentation"
|
26
|
+
Rake::RDocTask.new(:rdoc) do |rdoc|
|
27
|
+
rdoc.options << '--line-numbers' << '--inline-source' <<
|
28
|
+
'--main' << 'README' <<
|
29
|
+
'--title' << "Sliding Stats Documentation" <<
|
30
|
+
'--charset' << 'utf-8'
|
31
|
+
rdoc.rdoc_dir = "doc"
|
32
|
+
rdoc.rdoc_files.include 'README.rdoc'
|
33
|
+
rdoc.rdoc_files.include 'RDOX'
|
34
|
+
rdoc.rdoc_files.include("lib/sliding-stats/*.rb")
|
35
|
+
rdoc.rdoc_files.include("lib/sliding-stats/*/*.rb")
|
36
|
+
end
|
37
|
+
task :rdoc => ["RDOX"]
|
38
|
+
|
39
|
+
|
40
|
+
# PACKAGING =================================================================
|
41
|
+
|
42
|
+
# load gemspec like github's gem builder to surface any SAFE issues.
|
43
|
+
require 'rubygems/specification'
|
44
|
+
$spec = eval(File.read("sliding-stats.gemspec"))
|
45
|
+
|
46
|
+
def package(ext='')
|
47
|
+
"pkg/sliding-stats-#{$spec.version}" + ext
|
48
|
+
end
|
49
|
+
|
50
|
+
desc 'Build packages'
|
51
|
+
task :package => %w[.gem .tar.gz].map {|e| package(e)}
|
52
|
+
|
53
|
+
desc 'Build and install as local gem'
|
54
|
+
task :install => package('.gem') do
|
55
|
+
sh "gem install #{package('.gem')}"
|
56
|
+
end
|
57
|
+
|
58
|
+
directory 'pkg/'
|
59
|
+
|
60
|
+
file package('.gem') => %w[pkg/ sliding-stats.gemspec] + $spec.files do |f|
|
61
|
+
sh "gem build sliding-stats.gemspec"
|
62
|
+
mv File.basename(f.name), f.name
|
63
|
+
end
|
64
|
+
|
65
|
+
file package('.tar.gz') => %w[pkg/] + $spec.files do |f|
|
66
|
+
sh "git archive --format=tar HEAD | gzip > #{f.name}"
|
67
|
+
end
|
68
|
+
|
69
|
+
# desc 'Publish gem and tarball to rubyforge'
|
70
|
+
# task 'publish:gem' => [package('.gem'), package('.tar.gz')] do |t|
|
71
|
+
# sh < <-end
|
72
|
+
# rubyforge add_release rack rack-contrib #{$spec.version} #{package('.gem')} &&
|
73
|
+
# rubyforge add_file rack rack-contrib #{$spec.version} #{package('.tar.gz')}
|
74
|
+
# end
|
75
|
+
# end
|
76
|
+
|
77
|
+
# GEMSPEC ===================================================================
|
78
|
+
|
79
|
+
file 'sliding-stats.gemspec' => FileList['{lib,test}/**','Rakefile', 'README.rdoc'] do |f|
|
80
|
+
# read spec file and split out manifest section
|
81
|
+
spec = File.read(f.name)
|
82
|
+
parts = spec.split(" # = MANIFEST =\n")
|
83
|
+
fail 'bad spec' if parts.length != 3
|
84
|
+
# determine file list from git ls-files
|
85
|
+
files = `git ls-files`.
|
86
|
+
split("\n").sort.reject{ |file| file =~ /^\./ }.
|
87
|
+
map{ |file| " #{file}" }.join("\n")
|
88
|
+
# piece file back together and write...
|
89
|
+
parts[1] = " s.files = %w[\n#{files}\n ]\n"
|
90
|
+
spec = parts.join(" # = MANIFEST =\n")
|
91
|
+
spec.sub!(/s.date = '.*'/, "s.date = '#{Time.now.strftime("%Y-%m-%d")}'")
|
92
|
+
File.open(f.name, 'w') { |io| io.write(spec) }
|
93
|
+
puts "updated #{f.name}"
|
94
|
+
end
|
data/example/test.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
|
2
|
+
# This demonstrates how to configure the stats and generates an SVG of 1000 requests by page.
|
3
|
+
# The exclusion patterns are geared towards my website, and so you'd want to adapt them.
|
4
|
+
|
5
|
+
require 'sliding-stats'
|
6
|
+
|
7
|
+
opts = {
|
8
|
+
# The number of requests that is considered
|
9
|
+
:limit => 1000,
|
10
|
+
|
11
|
+
# If set to an integer, the number of requests between each time the data is persisted
|
12
|
+
# (using Marshal) to /var/tmp/slidingstats. You can provide a path by passing
|
13
|
+
# SlidingStats::Persist.new(number, path) instead, or you can provide any class that
|
14
|
+
# provides a #load and #save method -- see SlidingStats::Persist
|
15
|
+
:persist => nil,
|
16
|
+
|
17
|
+
# Pages where either the request or referrer match :ignore is not processed further,
|
18
|
+
# and doesn't count towards :limit
|
19
|
+
:ignore => [
|
20
|
+
/\.xml/, /\/feed/, /\.rdf/, /\.ico/, /\/static\//,/\/robots.txt/
|
21
|
+
/\/referers/, /\/stats.*/,
|
22
|
+
/http:\/\/search.live.com\/results.aspx/, # MSN referer spam
|
23
|
+
],
|
24
|
+
|
25
|
+
# Exclude entries from the referer graph and the referer to pages table
|
26
|
+
:exclude_referers => [
|
27
|
+
/http:\/\/www\.hokstad\.com/, # Not interested in seeing internal clicks
|
28
|
+
/^-/ # Direct traffic.
|
29
|
+
],
|
30
|
+
|
31
|
+
# Exclude entries from the page graph and referer to pages table.
|
32
|
+
:exclude_pages => [
|
33
|
+
],
|
34
|
+
|
35
|
+
# Rewrite referrer entries to make them more friendly, and group together
|
36
|
+
# referrers that don't have exactly the same URL
|
37
|
+
:rewrite_referers =>
|
38
|
+
[
|
39
|
+
[/http:\/\/.*\.google\..*?[?&]q=([^&]*)?&*.*/,"Google Search: '\\1'"],
|
40
|
+
[/http:\/\/www.google..*\/reader.*/,"Google Reader"]
|
41
|
+
]
|
42
|
+
}
|
43
|
+
|
44
|
+
view = SlidingStats::Controller.new(nil,"/stats")
|
45
|
+
window = SlidingStats::Window.new(view, opts)
|
46
|
+
|
47
|
+
# First we feed it stats from STDIN:
|
48
|
+
|
49
|
+
STDIN.each do |line|
|
50
|
+
line = line.split(" ")
|
51
|
+
window.call({"REQUEST_URI" => line[6],
|
52
|
+
"HTTP_REFERER" => line[10][1..-2]})
|
53
|
+
end
|
54
|
+
|
55
|
+
# Then we fake a stats request:
|
56
|
+
|
57
|
+
window.call({"REQUEST_URI" => "/stats/pages.svg"}).each do |line|
|
58
|
+
puts line
|
59
|
+
end
|
60
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
|
2
|
+
Feature: Maintain stats
|
3
|
+
In order to keep an eye on traffic the
|
4
|
+
Stats Module must keep track of
|
5
|
+
referrers and pages at all times.
|
6
|
+
|
7
|
+
Scenario Outline: Adding requests
|
8
|
+
Given there are <start> requests in the stats
|
9
|
+
When I add <requests> requests that are not excluded
|
10
|
+
Then there should be <total> pageviews
|
11
|
+
And there should be <total> referrers
|
12
|
+
|
13
|
+
Examples:
|
14
|
+
| start | requests | total |
|
15
|
+
| 5 | 1 | 6 |
|
16
|
+
| 10 | 23 | 33 |
|
17
|
+
| 0 | 5 | 5 |
|
18
|
+
|
19
|
+
Scenario Outline: Removing requests
|
20
|
+
Given there are <start> requests in the stats
|
21
|
+
When I remove <requests> requests that are not excluded
|
22
|
+
Then there should be <total> pageviews
|
23
|
+
And there should be <total> referrers
|
24
|
+
And there should be no pages rows with value 0
|
25
|
+
And there should be no referers rows with value 0
|
26
|
+
And there should be no rows with 0 in referers_to_pages
|
27
|
+
|
28
|
+
Examples:
|
29
|
+
| start | requests | total |
|
30
|
+
| 0 | 1 | 0 |
|
31
|
+
| 1 | 1 | 0 |
|
32
|
+
| 3 | 2 | 1 |
|
33
|
+
| 3 | 5 | 0 |
|
34
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
$: << File.expand_path(File.dirname(__FILE__)+"/../../lib/")
|
2
|
+
require 'sliding-stats'
|
3
|
+
require 'spec/expectations'
|
4
|
+
|
5
|
+
def valid_request
|
6
|
+
{"HTTP_REFERER" => "valid_referer",
|
7
|
+
"REQUEST_URI" => "valid_uri"}
|
8
|
+
end
|
9
|
+
|
10
|
+
Given /^there are (\d+) requests in the stats$/ do |n|
|
11
|
+
r = []
|
12
|
+
n.to_i.times { r << valid_request }
|
13
|
+
@stats = SlidingStats::Stats.new(r,{},{})
|
14
|
+
end
|
15
|
+
|
16
|
+
When /^I add (\d+) request[s]? that are not excluded$/ do |n|
|
17
|
+
n.to_i.times { @stats.add(valid_request) }
|
18
|
+
end
|
19
|
+
|
20
|
+
When /^I remove (\d+) request[s]? that are not excluded$/ do |n|
|
21
|
+
n.to_i.times { @stats.sub(valid_request) }
|
22
|
+
end
|
23
|
+
|
24
|
+
Then /^there should be (\d+) pageview[s]?$/ do |n|
|
25
|
+
@stats.pages.to_a.inject(0) {|s,a| s+a[1]}.should == n.to_i
|
26
|
+
end
|
27
|
+
|
28
|
+
Then /^there should be (\d+) referrer[s]?$/ do |n|
|
29
|
+
@stats.referers.to_a.inject(0) {|s,a| s+a[1]}.should == n.to_i
|
30
|
+
end
|
31
|
+
|
32
|
+
Then /^there should be no (\w+) rows with value (\d+)$/ do |r,n|
|
33
|
+
@stats.send(r.to_sym).to_a.detect {|k,v| v == n.to_i }.should == nil
|
34
|
+
end
|
35
|
+
|
36
|
+
Then /^there should be no rows with 0 in referers_to_pages$/ do
|
37
|
+
@stats.referers_to_pages.each do |r|
|
38
|
+
r[1].to_a.detect {|k,v| v == 0}.should == nil
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
$: << File.expand_path(File.dirname(__FILE__)+"/../../lib/")
|
2
|
+
require 'sliding-stats'
|
3
|
+
|
4
|
+
def valid_request
|
5
|
+
{"HTTP_REFERER" => "valid_referer",
|
6
|
+
"REQUEST_URI" => "valid_uri"}
|
7
|
+
end
|
8
|
+
|
9
|
+
Given /^there is a limit of (\d+) requests in the window$/ do |n|
|
10
|
+
@window = SlidingStats::Window.new(Proc.new {},
|
11
|
+
{:limit => n})
|
12
|
+
end
|
13
|
+
|
14
|
+
When /^I add (\d+) request[s]? that are not excluded to the window$/ do |n|
|
15
|
+
end
|
16
|
+
|
17
|
+
Then /^there should be (\d+) pageview[s]? in the window$/ do |n|
|
18
|
+
@window.stats.pages.to_a.inject(0) {|s,a| s+a[1]} == n.to_i
|
19
|
+
end
|
20
|
+
|
21
|
+
Then /^there should be (\d+) referrer[s]? in the window$/ do |n|
|
22
|
+
@window.stats.referers.to_a.inject(0) {|s,a| s+a[1]} == n.to_i
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
Feature: Maintain a sliding window
|
3
|
+
In order to keep an eye on traffic the
|
4
|
+
Window class must maintain a sliding window
|
5
|
+
with an upper size limit at all times.
|
6
|
+
|
7
|
+
Scenario Outline: Adding requests
|
8
|
+
Given there is a limit of <limit> requests in the window
|
9
|
+
When I add <requests> requests that are not excluded to the window
|
10
|
+
Then there should be <total> pageviews in the window
|
11
|
+
And there should be <total> referrers in the window
|
12
|
+
|
13
|
+
Examples:
|
14
|
+
| limit | requests | total |
|
15
|
+
| 0 | 1 | 0 |
|
16
|
+
| 5 | 3 | 3 |
|
17
|
+
| 5 | 5 | 5 |
|
18
|
+
| 5 | 10 | 5 |
|
19
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
|
2
|
+
require 'rack'
|
3
|
+
|
4
|
+
module SlidingStats
|
5
|
+
|
6
|
+
class Controller
|
7
|
+
def initialize app, opts
|
8
|
+
@app = app
|
9
|
+
@base = opts[:base] || "/stats"
|
10
|
+
@view = opts[:view] || View.new
|
11
|
+
@max_entries = opts[:max_entries] || 100
|
12
|
+
end
|
13
|
+
|
14
|
+
def call env
|
15
|
+
return Rack::Response.new("Missing 'slidingstats' object -- did you forget to set up SlidingStats::Window before SlidingStats::Controller ? ").finish if !env["slidingstats"]
|
16
|
+
|
17
|
+
uri = env["REQUEST_URI"]
|
18
|
+
@window = env["slidingstats"]
|
19
|
+
|
20
|
+
case uri
|
21
|
+
when @base
|
22
|
+
r_to_p = @window.stats.referers_to_pages.sort_by{|k,v| -v[:total]}[0..@max_entries-1]
|
23
|
+
referers = @window.stats.referers.sort_by{|k,v| -v}[0..@max_entries-1]
|
24
|
+
pages = @window.stats.pages.sort_by{|k,v| -v}[0..@max_entries-1]
|
25
|
+
return @view.show({:referers => referers, :pages => pages, :referers_to_pages => r_to_p, :base => @base})
|
26
|
+
when @base+"/referers.svg"
|
27
|
+
data = @window.stats.referers.sort_by{|k,v| -v}[0..@max_entries-1]
|
28
|
+
return @view.show_svg(data)
|
29
|
+
when @base+"/pages.svg"
|
30
|
+
data = @window.stats.pages.sort_by{|k,v| -v}[0..@max_entries-1]
|
31
|
+
return @view.show_svg(data)
|
32
|
+
else
|
33
|
+
return @app.call(env) if @app
|
34
|
+
return Rack::Response.new("(empty)").finish
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module SlidingStats
|
4
|
+
|
5
|
+
# This class provides basic persistence for SlidingStats
|
6
|
+
# To use it, simply add add :persist => [number of requests
|
7
|
+
# between saves] to the SlidingStats::Window options,
|
8
|
+
# or pass a different persistence class.
|
9
|
+
class Persist
|
10
|
+
def initialize every = 10,path="/var/tmp/slidingstats"
|
11
|
+
@every = every
|
12
|
+
@num = 0
|
13
|
+
@path = path
|
14
|
+
end
|
15
|
+
|
16
|
+
def load
|
17
|
+
begin
|
18
|
+
Marshal.load(File.read(@path))
|
19
|
+
rescue
|
20
|
+
[]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def save requests
|
25
|
+
@num += 1
|
26
|
+
if (@num % @every) == 0
|
27
|
+
File.open(@path,"w") do |f|
|
28
|
+
f.write(Marshal.dump(requests))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
|
2
|
+
module SlidingStats
|
3
|
+
|
4
|
+
# Calculates and maintains stats for a set of
|
5
|
+
# requests.
|
6
|
+
class Stats
|
7
|
+
attr_reader :referers, :pages, :referers_to_pages
|
8
|
+
def initialize request,ex_referers,ex_pages
|
9
|
+
@exclude_referers = ex_referers || []
|
10
|
+
@exclude_pages = ex_pages || []
|
11
|
+
|
12
|
+
@referers = {}
|
13
|
+
@pages = {}
|
14
|
+
@referers_to_pages = {} # Two level
|
15
|
+
|
16
|
+
request.each { |r| self.add(r) }
|
17
|
+
end
|
18
|
+
|
19
|
+
# Add a single line of stats data
|
20
|
+
def add r
|
21
|
+
ref = r["HTTP_REFERER"]
|
22
|
+
req = r["REQUEST_URI"]
|
23
|
+
|
24
|
+
ex_ref = @exclude_referers.detect{|pat| ref =~ pat}
|
25
|
+
ex_req = @exclude_pages.detect{|pat| req =~ pat}
|
26
|
+
|
27
|
+
if !ex_ref
|
28
|
+
@referers[ref] ||= 0
|
29
|
+
@referers[ref] += 1
|
30
|
+
end
|
31
|
+
|
32
|
+
if !ex_req
|
33
|
+
@pages[req] ||= 0
|
34
|
+
@pages[req] += 1
|
35
|
+
end
|
36
|
+
|
37
|
+
if !ex_ref && !ex_req
|
38
|
+
@referers_to_pages[ref] ||= {:total => 0}
|
39
|
+
@referers_to_pages[ref][req] ||= 0
|
40
|
+
@referers_to_pages[ref][req] += 1
|
41
|
+
@referers_to_pages[ref][:total] += 1
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def sub r
|
46
|
+
ref = r["HTTP_REFERER"]
|
47
|
+
req = r["REQUEST_URI"]
|
48
|
+
|
49
|
+
ex_ref = @exclude_referers.detect{|pat| ref =~ pat}
|
50
|
+
ex_req = @exclude_pages.detect{|pat| req =~ pat}
|
51
|
+
|
52
|
+
if !ex_ref && @referers[ref]
|
53
|
+
@referers[ref] -= 1
|
54
|
+
@referers.delete(ref) if @referers[ref] <= 0
|
55
|
+
end
|
56
|
+
|
57
|
+
if !ex_req && @pages[req]
|
58
|
+
@pages[req] -= 1
|
59
|
+
@pages.delete(req) if @pages[req] <= 0
|
60
|
+
end
|
61
|
+
|
62
|
+
if !ex_ref && !ex_req && @referers_to_pages[ref]
|
63
|
+
if @referers_to_pages[ref][req]
|
64
|
+
@referers_to_pages[ref][req] -= 1
|
65
|
+
@referers_to_pages[ref].delete(req) if @referers_to_pages[ref][req] <= 0
|
66
|
+
end
|
67
|
+
@referers_to_pages[ref][:total] -= 1
|
68
|
+
@referers_to_pages.delete(ref) if @referers_to_pages[ref][:total] <= 0
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'svg_graph'
|
2
|
+
require 'rack'
|
3
|
+
|
4
|
+
module SlidingStats
|
5
|
+
|
6
|
+
# Provides a basic view of the stats. You can easily provide a custom
|
7
|
+
# view by subclassing and overriding the #show method, or replacing it
|
8
|
+
# completely.
|
9
|
+
class View
|
10
|
+
FOOTER = <<-end_footer
|
11
|
+
</table>
|
12
|
+
<div style='margin-top: 50px'>Stats by <a href='http://www.hokstad.com/slidingstats'>Sliding Stats</a> -- Copyright 2009 <a href='http://www.hokstad.com/'>Vidar Hokstad</a>. </div>
|
13
|
+
</body></html>
|
14
|
+
end_footer
|
15
|
+
|
16
|
+
CSS = <<-end_css
|
17
|
+
h1, h2 { font-family: 'Lucida Sans Unicode', 'Lucida Grande', sans-serif; }
|
18
|
+
h2 { margin-top: 20px;}
|
19
|
+
|
20
|
+
table { display: inline; margin-top: 20px; margin-left: 100px; width: 90%; border: outset 1px grey;
|
21
|
+
background: #aaaaff; padding: 0px; align: left; text-align: left;
|
22
|
+
}
|
23
|
+
table.breakdown { background: #ccccff; margin-top: 1px; width: 100%; margin-left: 0px; padding: 5px; }
|
24
|
+
table.breakdown td.count { width: 40px; }
|
25
|
+
td.name { width: 50%; }
|
26
|
+
tr.odd { background: #aaaaff; }
|
27
|
+
tr.even { background: #bbbbff; }
|
28
|
+
end_css
|
29
|
+
|
30
|
+
def show(data)
|
31
|
+
r = Rack::Response.new
|
32
|
+
r.write("<html><head><title>Sliding Stats</title><style>" + CSS + "</style> <body>")
|
33
|
+
r.write("<h1>Sliding Stats</h1>")
|
34
|
+
# Setting the size here is a *hack*. Need to fix that
|
35
|
+
r.write("<h2>Most recent referrers</h2>")
|
36
|
+
r.write("<div style='width: 1000px;'><embed pluginspage=\"http://www.adobe.com/svg/viewer/install/\" type=\"image/svg+xml\" src=\"#{data[:base]}/referers.svg\" style=\"margin-left: 50px; width: 1000px; height: #{40 + 20*data[:referers].size}px;\"></div>")
|
37
|
+
r.write("<h2>Most recent pages</h2>")
|
38
|
+
r.write("<div style='width: 1000px;'><embed pluginspage=\"http://www.adobe.com/svg/viewer/install/\" type=\"image/svg+xml\" src=\"#{data[:base]}/pages.svg\" style=\"margin-left: 50px; width: 1000px; height: #{40 + 20*data[:pages].size}px;\"></div>")
|
39
|
+
r.write("<h2>Most recent referrers broken down by pages</h2>")
|
40
|
+
r.write("<table><tr><th>Referer</th><th>Pages</th></tr>\n")
|
41
|
+
odd = true
|
42
|
+
data[:referers_to_pages].each do |k,v|
|
43
|
+
k = k[0..79] + "..." if k.length > 80
|
44
|
+
r.write("<tr class='#{odd ? 'odd':'even'}'><td class='name'>#{CGI.escapeHTML(k)}</td> <td><table class='breakdown'>")
|
45
|
+
total = v[:total]
|
46
|
+
if v.size > 2 # include :total
|
47
|
+
r.write("<tr><td class='count'>#{total}</td><td><strong>total</strong></td></tr>")
|
48
|
+
end
|
49
|
+
v.sort_by{|page,count| -count}.each do |page,count|
|
50
|
+
r.write("<tr><td class='count'>#{count}</td><td>#{page.to_s}</td></tr>") if page != :total
|
51
|
+
end
|
52
|
+
r.write("</table></td></tr>\n")
|
53
|
+
odd = !odd
|
54
|
+
end
|
55
|
+
r.write(FOOTER)
|
56
|
+
r.finish
|
57
|
+
end
|
58
|
+
|
59
|
+
def show_svg(src)
|
60
|
+
fields = []
|
61
|
+
data = []
|
62
|
+
src.each do |k,v|
|
63
|
+
if k != "-" # Excluding because of referers
|
64
|
+
k = k[0..79] + "..." if k.length > 80
|
65
|
+
fields << CGI.escapeHTML(k)
|
66
|
+
data << v
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
if fields.empty?
|
71
|
+
r = Rack::Response.new("No data")
|
72
|
+
return r.finish
|
73
|
+
end
|
74
|
+
|
75
|
+
graph = SVG::Graph::BarHorizontal.new(
|
76
|
+
:height => 40 + 20 * data.size,
|
77
|
+
:width => 1000,
|
78
|
+
:fields => fields.reverse
|
79
|
+
)
|
80
|
+
graph.add_data(:data => data.reverse)
|
81
|
+
graph.rotate_y_labels = false
|
82
|
+
graph.scale_integers = true
|
83
|
+
graph.key = false
|
84
|
+
r = Rack::Response.new
|
85
|
+
r["Content-Type"] = "image/svg+xml"
|
86
|
+
r.write(graph.burn)
|
87
|
+
r.finish
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
|
2
|
+
require 'sliding-stats/stats'
|
3
|
+
require 'sliding-stats/persist'
|
4
|
+
require 'cgi'
|
5
|
+
|
6
|
+
module SlidingStats
|
7
|
+
DEFAULT_WINDOW = 500
|
8
|
+
|
9
|
+
# Provides a "sliding window" over the stats. You provide
|
10
|
+
# a limit, and then feeds data into it. When the number of
|
11
|
+
# lines of data exceeds the limit, the oldest gets removed.
|
12
|
+
#
|
13
|
+
# The actual stats calculation is handled by the Stats class
|
14
|
+
#
|
15
|
+
# At any point you can extract stats from from the current
|
16
|
+
# window.
|
17
|
+
#
|
18
|
+
# The following options can be passed in the opts argument:
|
19
|
+
# * :limit => the number of stats lines to keep
|
20
|
+
# * :ignore => Requests where this matches *either* the referer *or* the request
|
21
|
+
# *or* the user agent will not be considered at all.
|
22
|
+
# * :request_methods => Array of HTTP methods to track. Defaults to :get
|
23
|
+
# as POST, PUT etc. on "normal" sites rarely happen
|
24
|
+
# on inbound referrals, and so we'd be likely to overcount
|
25
|
+
# access to a specific page
|
26
|
+
# * :exclude_[referers|pages] => Arrays that will be matched against
|
27
|
+
# REQUEST_URI and HTTP_REFERER to decide
|
28
|
+
# whether or not to exclude this request from
|
29
|
+
# the appropriate stats.
|
30
|
+
# * :rewrite_referer =>
|
31
|
+
# An Array of arrays consisting of regexps
|
32
|
+
# and a rewrite pattern to filter the
|
33
|
+
# HTTP_REFERER against
|
34
|
+
class Window
|
35
|
+
attr_reader :stats
|
36
|
+
|
37
|
+
def initialize app, opts = {}
|
38
|
+
@app = app
|
39
|
+
@limit = (opts[:limit] || DEFAULT_WINDOW).to_i
|
40
|
+
@exclude_referers = opts[:exclude_referers] || []
|
41
|
+
@rewrite_referers = opts[:rewrite_referers] || []
|
42
|
+
@request_methods = opts[:request_methods] || [:get]
|
43
|
+
@exclude_pages = opts[:exclude_pages] || []
|
44
|
+
@ignore = opts[:ignore] || []
|
45
|
+
@persist = opts[:persist]
|
46
|
+
|
47
|
+
@requests = []
|
48
|
+
if @persist.is_a?(Numeric)
|
49
|
+
@persist = SlidingStats::Persist.new(@persist)
|
50
|
+
@requests = @persist.load
|
51
|
+
end
|
52
|
+
@stats = Stats.new(@requests,@exclude_referers,@exclude_pages)
|
53
|
+
end
|
54
|
+
|
55
|
+
def call env
|
56
|
+
ref = env["HTTP_REFERER"] || "-"
|
57
|
+
req = env["REQUEST_URI"]
|
58
|
+
ua = env["HTTP_USER_AGENT"]
|
59
|
+
req_meth = env["REQUEST_METHOD"].downcase.to_sym
|
60
|
+
|
61
|
+
if @request_methods.include?(req_meth) && !@ignore.detect{|pat| ref =~ pat || req =~ pat || ua =~ pat}
|
62
|
+
newref = @rewrite_referers.inject(ref) { |nr,r| nr.gsub(r[0],r[1]) }
|
63
|
+
ref = CGI.unescape(newref) if newref != ref
|
64
|
+
|
65
|
+
stats = {
|
66
|
+
"HTTP_REFERER" => ref,
|
67
|
+
"REQUEST_URI" => req
|
68
|
+
}
|
69
|
+
@requests << stats
|
70
|
+
@stats.add(stats)
|
71
|
+
while @requests.size > @limit
|
72
|
+
@stats.sub(@requests.shift)
|
73
|
+
end
|
74
|
+
@persist.save(@requests) if @persist
|
75
|
+
end
|
76
|
+
|
77
|
+
env["slidingstats"] = self
|
78
|
+
@app.call(env)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
|
2
|
+
Gem::Specification.new do |s|
|
3
|
+
s.specification_version = 2 if s.respond_to? :specification_version=
|
4
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
5
|
+
|
6
|
+
s.name = 'sliding-stats'
|
7
|
+
s.version = '0.2.8'
|
8
|
+
s.date = '2011-10-14'
|
9
|
+
|
10
|
+
s.description = "Rack Middleware to provide a 'sliding view' over the last N requests to your web app"
|
11
|
+
s.summary = s.description
|
12
|
+
|
13
|
+
s.authors = ["vidarh"]
|
14
|
+
s.email = "vidar@hokstad.com"
|
15
|
+
|
16
|
+
# = MANIFEST =
|
17
|
+
s.files = %w[
|
18
|
+
README.rdoc
|
19
|
+
Rakefile
|
20
|
+
example/test.rb
|
21
|
+
features/stats.feature
|
22
|
+
features/step_definitions/stats_steps.rb
|
23
|
+
features/step_definitions/window_steps.rb
|
24
|
+
features/window.feature
|
25
|
+
lib/sliding-stats.rb
|
26
|
+
lib/sliding-stats/controller.rb
|
27
|
+
lib/sliding-stats/persist.rb
|
28
|
+
lib/sliding-stats/stats.rb
|
29
|
+
lib/sliding-stats/view.rb
|
30
|
+
lib/sliding-stats/window.rb
|
31
|
+
sliding-stats.gemspec
|
32
|
+
]
|
33
|
+
# = MANIFEST =
|
34
|
+
|
35
|
+
s.test_files = s.files.select {|path| path =~ /^test\/spec_.*\.rb/}
|
36
|
+
|
37
|
+
s.extra_rdoc_files = %w[]
|
38
|
+
s.add_dependency 'rack', '>= 0.9.1'
|
39
|
+
s.add_dependency 'svg_graph', '>= 0.7'
|
40
|
+
#s.add_development_dependency 'json', '>= 1.1'
|
41
|
+
|
42
|
+
s.has_rdoc = true
|
43
|
+
s.homepage = "http://www.hokstad.com/slidingstats"
|
44
|
+
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "slidingstats", "--main", "README.rdoc"]
|
45
|
+
s.require_paths = %w[lib]
|
46
|
+
s.rubygems_version = '1.1.1'
|
47
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sliding-stats
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 7
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 8
|
10
|
+
version: 0.2.8
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- vidarh
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-10-14 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: rack
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 57
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
- 9
|
32
|
+
- 1
|
33
|
+
version: 0.9.1
|
34
|
+
type: :runtime
|
35
|
+
version_requirements: *id001
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: svg_graph
|
38
|
+
prerelease: false
|
39
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
hash: 5
|
45
|
+
segments:
|
46
|
+
- 0
|
47
|
+
- 7
|
48
|
+
version: "0.7"
|
49
|
+
type: :runtime
|
50
|
+
version_requirements: *id002
|
51
|
+
description: Rack Middleware to provide a 'sliding view' over the last N requests to your web app
|
52
|
+
email: vidar@hokstad.com
|
53
|
+
executables: []
|
54
|
+
|
55
|
+
extensions: []
|
56
|
+
|
57
|
+
extra_rdoc_files: []
|
58
|
+
|
59
|
+
files:
|
60
|
+
- README.rdoc
|
61
|
+
- Rakefile
|
62
|
+
- example/test.rb
|
63
|
+
- features/stats.feature
|
64
|
+
- features/step_definitions/stats_steps.rb
|
65
|
+
- features/step_definitions/window_steps.rb
|
66
|
+
- features/window.feature
|
67
|
+
- lib/sliding-stats.rb
|
68
|
+
- lib/sliding-stats/controller.rb
|
69
|
+
- lib/sliding-stats/persist.rb
|
70
|
+
- lib/sliding-stats/stats.rb
|
71
|
+
- lib/sliding-stats/view.rb
|
72
|
+
- lib/sliding-stats/window.rb
|
73
|
+
- sliding-stats.gemspec
|
74
|
+
homepage: http://www.hokstad.com/slidingstats
|
75
|
+
licenses: []
|
76
|
+
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options:
|
79
|
+
- --line-numbers
|
80
|
+
- --inline-source
|
81
|
+
- --title
|
82
|
+
- slidingstats
|
83
|
+
- --main
|
84
|
+
- README.rdoc
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
none: false
|
89
|
+
requirements:
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
hash: 3
|
93
|
+
segments:
|
94
|
+
- 0
|
95
|
+
version: "0"
|
96
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
hash: 3
|
102
|
+
segments:
|
103
|
+
- 0
|
104
|
+
version: "0"
|
105
|
+
requirements: []
|
106
|
+
|
107
|
+
rubyforge_project:
|
108
|
+
rubygems_version: 1.8.6
|
109
|
+
signing_key:
|
110
|
+
specification_version: 2
|
111
|
+
summary: Rack Middleware to provide a 'sliding view' over the last N requests to your web app
|
112
|
+
test_files: []
|
113
|
+
|