sliding-stats 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +0 -0
- data/Rakefile +94 -0
- data/example/test.rb +60 -0
- data/features/stats.feature +34 -0
- data/features/step_definitions/stats_steps.rb +40 -0
- data/features/step_definitions/window_steps.rb +24 -0
- data/features/window.feature +19 -0
- data/lib/sliding-stats.rb +6 -0
- data/lib/sliding-stats/controller.rb +38 -0
- data/lib/sliding-stats/persist.rb +33 -0
- data/lib/sliding-stats/stats.rb +72 -0
- data/lib/sliding-stats/view.rb +90 -0
- data/lib/sliding-stats/window.rb +81 -0
- data/sliding-stats.gemspec +47 -0
- metadata +113 -0
data/README.rdoc
ADDED
File without changes
|
data/Rakefile
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
# Rakefile for SlidingStats. -*-ruby-*-
|
2
|
+
# Shamelessly stolen from Rack::Contrib
|
3
|
+
|
4
|
+
require 'rake/rdoctask'
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
desc "Run all the tests"
|
8
|
+
#task :default => [:test]
|
9
|
+
|
10
|
+
#desc "Generate RDox"
|
11
|
+
#task "RDOX" do
|
12
|
+
# sh "specrb -Ilib:test -a --rdox >RDOX"
|
13
|
+
#end
|
14
|
+
|
15
|
+
#desc "Run all the fast tests"
|
16
|
+
#task :test do
|
17
|
+
# sh "specrb -Ilib:test -w #{ENV['TEST'] || '-a'} #{ENV['TESTOPTS']}"
|
18
|
+
#end
|
19
|
+
|
20
|
+
#desc "Run all the tests"
|
21
|
+
#task :fulltest do
|
22
|
+
# sh "specrb -Ilib:test -w #{ENV['TEST'] || '-a'} #{ENV['TESTOPTS']}"
|
23
|
+
#end
|
24
|
+
|
25
|
+
desc "Generate RDoc documentation"
|
26
|
+
Rake::RDocTask.new(:rdoc) do |rdoc|
|
27
|
+
rdoc.options << '--line-numbers' << '--inline-source' <<
|
28
|
+
'--main' << 'README' <<
|
29
|
+
'--title' << "Sliding Stats Documentation" <<
|
30
|
+
'--charset' << 'utf-8'
|
31
|
+
rdoc.rdoc_dir = "doc"
|
32
|
+
rdoc.rdoc_files.include 'README.rdoc'
|
33
|
+
rdoc.rdoc_files.include 'RDOX'
|
34
|
+
rdoc.rdoc_files.include("lib/sliding-stats/*.rb")
|
35
|
+
rdoc.rdoc_files.include("lib/sliding-stats/*/*.rb")
|
36
|
+
end
|
37
|
+
task :rdoc => ["RDOX"]
|
38
|
+
|
39
|
+
|
40
|
+
# PACKAGING =================================================================
|
41
|
+
|
42
|
+
# load gemspec like github's gem builder to surface any SAFE issues.
|
43
|
+
require 'rubygems/specification'
|
44
|
+
$spec = eval(File.read("sliding-stats.gemspec"))
|
45
|
+
|
46
|
+
def package(ext='')
|
47
|
+
"pkg/sliding-stats-#{$spec.version}" + ext
|
48
|
+
end
|
49
|
+
|
50
|
+
desc 'Build packages'
|
51
|
+
task :package => %w[.gem .tar.gz].map {|e| package(e)}
|
52
|
+
|
53
|
+
desc 'Build and install as local gem'
|
54
|
+
task :install => package('.gem') do
|
55
|
+
sh "gem install #{package('.gem')}"
|
56
|
+
end
|
57
|
+
|
58
|
+
directory 'pkg/'
|
59
|
+
|
60
|
+
file package('.gem') => %w[pkg/ sliding-stats.gemspec] + $spec.files do |f|
|
61
|
+
sh "gem build sliding-stats.gemspec"
|
62
|
+
mv File.basename(f.name), f.name
|
63
|
+
end
|
64
|
+
|
65
|
+
file package('.tar.gz') => %w[pkg/] + $spec.files do |f|
|
66
|
+
sh "git archive --format=tar HEAD | gzip > #{f.name}"
|
67
|
+
end
|
68
|
+
|
69
|
+
# desc 'Publish gem and tarball to rubyforge'
|
70
|
+
# task 'publish:gem' => [package('.gem'), package('.tar.gz')] do |t|
|
71
|
+
# sh < <-end
|
72
|
+
# rubyforge add_release rack rack-contrib #{$spec.version} #{package('.gem')} &&
|
73
|
+
# rubyforge add_file rack rack-contrib #{$spec.version} #{package('.tar.gz')}
|
74
|
+
# end
|
75
|
+
# end
|
76
|
+
|
77
|
+
# GEMSPEC ===================================================================
|
78
|
+
|
79
|
+
file 'sliding-stats.gemspec' => FileList['{lib,test}/**','Rakefile', 'README.rdoc'] do |f|
|
80
|
+
# read spec file and split out manifest section
|
81
|
+
spec = File.read(f.name)
|
82
|
+
parts = spec.split(" # = MANIFEST =\n")
|
83
|
+
fail 'bad spec' if parts.length != 3
|
84
|
+
# determine file list from git ls-files
|
85
|
+
files = `git ls-files`.
|
86
|
+
split("\n").sort.reject{ |file| file =~ /^\./ }.
|
87
|
+
map{ |file| " #{file}" }.join("\n")
|
88
|
+
# piece file back together and write...
|
89
|
+
parts[1] = " s.files = %w[\n#{files}\n ]\n"
|
90
|
+
spec = parts.join(" # = MANIFEST =\n")
|
91
|
+
spec.sub!(/s.date = '.*'/, "s.date = '#{Time.now.strftime("%Y-%m-%d")}'")
|
92
|
+
File.open(f.name, 'w') { |io| io.write(spec) }
|
93
|
+
puts "updated #{f.name}"
|
94
|
+
end
|
data/example/test.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
|
2
|
+
# This demonstrates how to configure the stats and generates an SVG of 1000 requests by page.
|
3
|
+
# The exclusion patterns are geared towards my website, and so you'd want to adapt them.
|
4
|
+
|
5
|
+
require 'sliding-stats'
|
6
|
+
|
7
|
+
opts = {
|
8
|
+
# The number of requests that is considered
|
9
|
+
:limit => 1000,
|
10
|
+
|
11
|
+
# If set to an integer, the number of requests between each time the data is persisted
|
12
|
+
# (using Marshal) to /var/tmp/slidingstats. You can provide a path by passing
|
13
|
+
# SlidingStats::Persist.new(number, path) instead, or you can provide any class that
|
14
|
+
# provides a #load and #save method -- see SlidingStats::Persist
|
15
|
+
:persist => nil,
|
16
|
+
|
17
|
+
# Pages where either the request or referrer match :ignore is not processed further,
|
18
|
+
# and doesn't count towards :limit
|
19
|
+
:ignore => [
|
20
|
+
/\.xml/, /\/feed/, /\.rdf/, /\.ico/, /\/static\//,/\/robots.txt/
|
21
|
+
/\/referers/, /\/stats.*/,
|
22
|
+
/http:\/\/search.live.com\/results.aspx/, # MSN referer spam
|
23
|
+
],
|
24
|
+
|
25
|
+
# Exclude entries from the referer graph and the referer to pages table
|
26
|
+
:exclude_referers => [
|
27
|
+
/http:\/\/www\.hokstad\.com/, # Not interested in seeing internal clicks
|
28
|
+
/^-/ # Direct traffic.
|
29
|
+
],
|
30
|
+
|
31
|
+
# Exclude entries from the page graph and referer to pages table.
|
32
|
+
:exclude_pages => [
|
33
|
+
],
|
34
|
+
|
35
|
+
# Rewrite referrer entries to make them more friendly, and group together
|
36
|
+
# referrers that don't have exactly the same URL
|
37
|
+
:rewrite_referers =>
|
38
|
+
[
|
39
|
+
[/http:\/\/.*\.google\..*?[?&]q=([^&]*)?&*.*/,"Google Search: '\\1'"],
|
40
|
+
[/http:\/\/www.google..*\/reader.*/,"Google Reader"]
|
41
|
+
]
|
42
|
+
}
|
43
|
+
|
44
|
+
view = SlidingStats::Controller.new(nil,"/stats")
|
45
|
+
window = SlidingStats::Window.new(view, opts)
|
46
|
+
|
47
|
+
# First we feed it stats from STDIN:
|
48
|
+
|
49
|
+
STDIN.each do |line|
|
50
|
+
line = line.split(" ")
|
51
|
+
window.call({"REQUEST_URI" => line[6],
|
52
|
+
"HTTP_REFERER" => line[10][1..-2]})
|
53
|
+
end
|
54
|
+
|
55
|
+
# Then we fake a stats request:
|
56
|
+
|
57
|
+
window.call({"REQUEST_URI" => "/stats/pages.svg"}).each do |line|
|
58
|
+
puts line
|
59
|
+
end
|
60
|
+
|
@@ -0,0 +1,34 @@
|
|
1
|
+
|
2
|
+
Feature: Maintain stats
|
3
|
+
In order to keep an eye on traffic the
|
4
|
+
Stats Module must keep track of
|
5
|
+
referrers and pages at all times.
|
6
|
+
|
7
|
+
Scenario Outline: Adding requests
|
8
|
+
Given there are <start> requests in the stats
|
9
|
+
When I add <requests> requests that are not excluded
|
10
|
+
Then there should be <total> pageviews
|
11
|
+
And there should be <total> referrers
|
12
|
+
|
13
|
+
Examples:
|
14
|
+
| start | requests | total |
|
15
|
+
| 5 | 1 | 6 |
|
16
|
+
| 10 | 23 | 33 |
|
17
|
+
| 0 | 5 | 5 |
|
18
|
+
|
19
|
+
Scenario Outline: Removing requests
|
20
|
+
Given there are <start> requests in the stats
|
21
|
+
When I remove <requests> requests that are not excluded
|
22
|
+
Then there should be <total> pageviews
|
23
|
+
And there should be <total> referrers
|
24
|
+
And there should be no pages rows with value 0
|
25
|
+
And there should be no referers rows with value 0
|
26
|
+
And there should be no rows with 0 in referers_to_pages
|
27
|
+
|
28
|
+
Examples:
|
29
|
+
| start | requests | total |
|
30
|
+
| 0 | 1 | 0 |
|
31
|
+
| 1 | 1 | 0 |
|
32
|
+
| 3 | 2 | 1 |
|
33
|
+
| 3 | 5 | 0 |
|
34
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
$: << File.expand_path(File.dirname(__FILE__)+"/../../lib/")
|
2
|
+
require 'sliding-stats'
|
3
|
+
require 'spec/expectations'
|
4
|
+
|
5
|
+
def valid_request
|
6
|
+
{"HTTP_REFERER" => "valid_referer",
|
7
|
+
"REQUEST_URI" => "valid_uri"}
|
8
|
+
end
|
9
|
+
|
10
|
+
Given /^there are (\d+) requests in the stats$/ do |n|
|
11
|
+
r = []
|
12
|
+
n.to_i.times { r << valid_request }
|
13
|
+
@stats = SlidingStats::Stats.new(r,{},{})
|
14
|
+
end
|
15
|
+
|
16
|
+
When /^I add (\d+) request[s]? that are not excluded$/ do |n|
|
17
|
+
n.to_i.times { @stats.add(valid_request) }
|
18
|
+
end
|
19
|
+
|
20
|
+
When /^I remove (\d+) request[s]? that are not excluded$/ do |n|
|
21
|
+
n.to_i.times { @stats.sub(valid_request) }
|
22
|
+
end
|
23
|
+
|
24
|
+
Then /^there should be (\d+) pageview[s]?$/ do |n|
|
25
|
+
@stats.pages.to_a.inject(0) {|s,a| s+a[1]}.should == n.to_i
|
26
|
+
end
|
27
|
+
|
28
|
+
Then /^there should be (\d+) referrer[s]?$/ do |n|
|
29
|
+
@stats.referers.to_a.inject(0) {|s,a| s+a[1]}.should == n.to_i
|
30
|
+
end
|
31
|
+
|
32
|
+
Then /^there should be no (\w+) rows with value (\d+)$/ do |r,n|
|
33
|
+
@stats.send(r.to_sym).to_a.detect {|k,v| v == n.to_i }.should == nil
|
34
|
+
end
|
35
|
+
|
36
|
+
Then /^there should be no rows with 0 in referers_to_pages$/ do
|
37
|
+
@stats.referers_to_pages.each do |r|
|
38
|
+
r[1].to_a.detect {|k,v| v == 0}.should == nil
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
$: << File.expand_path(File.dirname(__FILE__)+"/../../lib/")
|
2
|
+
require 'sliding-stats'
|
3
|
+
|
4
|
+
def valid_request
|
5
|
+
{"HTTP_REFERER" => "valid_referer",
|
6
|
+
"REQUEST_URI" => "valid_uri"}
|
7
|
+
end
|
8
|
+
|
9
|
+
Given /^there is a limit of (\d+) requests in the window$/ do |n|
|
10
|
+
@window = SlidingStats::Window.new(Proc.new {},
|
11
|
+
{:limit => n})
|
12
|
+
end
|
13
|
+
|
14
|
+
When /^I add (\d+) request[s]? that are not excluded to the window$/ do |n|
|
15
|
+
end
|
16
|
+
|
17
|
+
Then /^there should be (\d+) pageview[s]? in the window$/ do |n|
|
18
|
+
@window.stats.pages.to_a.inject(0) {|s,a| s+a[1]} == n.to_i
|
19
|
+
end
|
20
|
+
|
21
|
+
Then /^there should be (\d+) referrer[s]? in the window$/ do |n|
|
22
|
+
@window.stats.referers.to_a.inject(0) {|s,a| s+a[1]} == n.to_i
|
23
|
+
end
|
24
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
Feature: Maintain a sliding window
|
3
|
+
In order to keep an eye on traffic the
|
4
|
+
Window class must maintain a sliding window
|
5
|
+
with an upper size limit at all times.
|
6
|
+
|
7
|
+
Scenario Outline: Adding requests
|
8
|
+
Given there is a limit of <limit> requests in the window
|
9
|
+
When I add <requests> requests that are not excluded to the window
|
10
|
+
Then there should be <total> pageviews in the window
|
11
|
+
And there should be <total> referrers in the window
|
12
|
+
|
13
|
+
Examples:
|
14
|
+
| limit | requests | total |
|
15
|
+
| 0 | 1 | 0 |
|
16
|
+
| 5 | 3 | 3 |
|
17
|
+
| 5 | 5 | 5 |
|
18
|
+
| 5 | 10 | 5 |
|
19
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
|
2
|
+
require 'rack'
|
3
|
+
|
4
|
+
module SlidingStats
|
5
|
+
|
6
|
+
class Controller
|
7
|
+
def initialize app, opts
|
8
|
+
@app = app
|
9
|
+
@base = opts[:base] || "/stats"
|
10
|
+
@view = opts[:view] || View.new
|
11
|
+
@max_entries = opts[:max_entries] || 100
|
12
|
+
end
|
13
|
+
|
14
|
+
def call env
|
15
|
+
return Rack::Response.new("Missing 'slidingstats' object -- did you forget to set up SlidingStats::Window before SlidingStats::Controller ? ").finish if !env["slidingstats"]
|
16
|
+
|
17
|
+
uri = env["REQUEST_URI"]
|
18
|
+
@window = env["slidingstats"]
|
19
|
+
|
20
|
+
case uri
|
21
|
+
when @base
|
22
|
+
r_to_p = @window.stats.referers_to_pages.sort_by{|k,v| -v[:total]}[0..@max_entries-1]
|
23
|
+
referers = @window.stats.referers.sort_by{|k,v| -v}[0..@max_entries-1]
|
24
|
+
pages = @window.stats.pages.sort_by{|k,v| -v}[0..@max_entries-1]
|
25
|
+
return @view.show({:referers => referers, :pages => pages, :referers_to_pages => r_to_p, :base => @base})
|
26
|
+
when @base+"/referers.svg"
|
27
|
+
data = @window.stats.referers.sort_by{|k,v| -v}[0..@max_entries-1]
|
28
|
+
return @view.show_svg(data)
|
29
|
+
when @base+"/pages.svg"
|
30
|
+
data = @window.stats.pages.sort_by{|k,v| -v}[0..@max_entries-1]
|
31
|
+
return @view.show_svg(data)
|
32
|
+
else
|
33
|
+
return @app.call(env) if @app
|
34
|
+
return Rack::Response.new("(empty)").finish
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
module SlidingStats
|
4
|
+
|
5
|
+
# This class provides basic persistence for SlidingStats
|
6
|
+
# To use it, simply add add :persist => [number of requests
|
7
|
+
# between saves] to the SlidingStats::Window options,
|
8
|
+
# or pass a different persistence class.
|
9
|
+
class Persist
|
10
|
+
def initialize every = 10,path="/var/tmp/slidingstats"
|
11
|
+
@every = every
|
12
|
+
@num = 0
|
13
|
+
@path = path
|
14
|
+
end
|
15
|
+
|
16
|
+
def load
|
17
|
+
begin
|
18
|
+
Marshal.load(File.read(@path))
|
19
|
+
rescue
|
20
|
+
[]
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def save requests
|
25
|
+
@num += 1
|
26
|
+
if (@num % @every) == 0
|
27
|
+
File.open(@path,"w") do |f|
|
28
|
+
f.write(Marshal.dump(requests))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
|
2
|
+
module SlidingStats
|
3
|
+
|
4
|
+
# Calculates and maintains stats for a set of
|
5
|
+
# requests.
|
6
|
+
class Stats
|
7
|
+
attr_reader :referers, :pages, :referers_to_pages
|
8
|
+
def initialize request,ex_referers,ex_pages
|
9
|
+
@exclude_referers = ex_referers || []
|
10
|
+
@exclude_pages = ex_pages || []
|
11
|
+
|
12
|
+
@referers = {}
|
13
|
+
@pages = {}
|
14
|
+
@referers_to_pages = {} # Two level
|
15
|
+
|
16
|
+
request.each { |r| self.add(r) }
|
17
|
+
end
|
18
|
+
|
19
|
+
# Add a single line of stats data
|
20
|
+
def add r
|
21
|
+
ref = r["HTTP_REFERER"]
|
22
|
+
req = r["REQUEST_URI"]
|
23
|
+
|
24
|
+
ex_ref = @exclude_referers.detect{|pat| ref =~ pat}
|
25
|
+
ex_req = @exclude_pages.detect{|pat| req =~ pat}
|
26
|
+
|
27
|
+
if !ex_ref
|
28
|
+
@referers[ref] ||= 0
|
29
|
+
@referers[ref] += 1
|
30
|
+
end
|
31
|
+
|
32
|
+
if !ex_req
|
33
|
+
@pages[req] ||= 0
|
34
|
+
@pages[req] += 1
|
35
|
+
end
|
36
|
+
|
37
|
+
if !ex_ref && !ex_req
|
38
|
+
@referers_to_pages[ref] ||= {:total => 0}
|
39
|
+
@referers_to_pages[ref][req] ||= 0
|
40
|
+
@referers_to_pages[ref][req] += 1
|
41
|
+
@referers_to_pages[ref][:total] += 1
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def sub r
|
46
|
+
ref = r["HTTP_REFERER"]
|
47
|
+
req = r["REQUEST_URI"]
|
48
|
+
|
49
|
+
ex_ref = @exclude_referers.detect{|pat| ref =~ pat}
|
50
|
+
ex_req = @exclude_pages.detect{|pat| req =~ pat}
|
51
|
+
|
52
|
+
if !ex_ref && @referers[ref]
|
53
|
+
@referers[ref] -= 1
|
54
|
+
@referers.delete(ref) if @referers[ref] <= 0
|
55
|
+
end
|
56
|
+
|
57
|
+
if !ex_req && @pages[req]
|
58
|
+
@pages[req] -= 1
|
59
|
+
@pages.delete(req) if @pages[req] <= 0
|
60
|
+
end
|
61
|
+
|
62
|
+
if !ex_ref && !ex_req && @referers_to_pages[ref]
|
63
|
+
if @referers_to_pages[ref][req]
|
64
|
+
@referers_to_pages[ref][req] -= 1
|
65
|
+
@referers_to_pages[ref].delete(req) if @referers_to_pages[ref][req] <= 0
|
66
|
+
end
|
67
|
+
@referers_to_pages[ref][:total] -= 1
|
68
|
+
@referers_to_pages.delete(ref) if @referers_to_pages[ref][:total] <= 0
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require 'svg_graph'
|
2
|
+
require 'rack'
|
3
|
+
|
4
|
+
module SlidingStats
|
5
|
+
|
6
|
+
# Provides a basic view of the stats. You can easily provide a custom
|
7
|
+
# view by subclassing and overriding the #show method, or replacing it
|
8
|
+
# completely.
|
9
|
+
class View
|
10
|
+
FOOTER = <<-end_footer
|
11
|
+
</table>
|
12
|
+
<div style='margin-top: 50px'>Stats by <a href='http://www.hokstad.com/slidingstats'>Sliding Stats</a> -- Copyright 2009 <a href='http://www.hokstad.com/'>Vidar Hokstad</a>. </div>
|
13
|
+
</body></html>
|
14
|
+
end_footer
|
15
|
+
|
16
|
+
CSS = <<-end_css
|
17
|
+
h1, h2 { font-family: 'Lucida Sans Unicode', 'Lucida Grande', sans-serif; }
|
18
|
+
h2 { margin-top: 20px;}
|
19
|
+
|
20
|
+
table { display: inline; margin-top: 20px; margin-left: 100px; width: 90%; border: outset 1px grey;
|
21
|
+
background: #aaaaff; padding: 0px; align: left; text-align: left;
|
22
|
+
}
|
23
|
+
table.breakdown { background: #ccccff; margin-top: 1px; width: 100%; margin-left: 0px; padding: 5px; }
|
24
|
+
table.breakdown td.count { width: 40px; }
|
25
|
+
td.name { width: 50%; }
|
26
|
+
tr.odd { background: #aaaaff; }
|
27
|
+
tr.even { background: #bbbbff; }
|
28
|
+
end_css
|
29
|
+
|
30
|
+
def show(data)
|
31
|
+
r = Rack::Response.new
|
32
|
+
r.write("<html><head><title>Sliding Stats</title><style>" + CSS + "</style> <body>")
|
33
|
+
r.write("<h1>Sliding Stats</h1>")
|
34
|
+
# Setting the size here is a *hack*. Need to fix that
|
35
|
+
r.write("<h2>Most recent referrers</h2>")
|
36
|
+
r.write("<div style='width: 1000px;'><embed pluginspage=\"http://www.adobe.com/svg/viewer/install/\" type=\"image/svg+xml\" src=\"#{data[:base]}/referers.svg\" style=\"margin-left: 50px; width: 1000px; height: #{40 + 20*data[:referers].size}px;\"></div>")
|
37
|
+
r.write("<h2>Most recent pages</h2>")
|
38
|
+
r.write("<div style='width: 1000px;'><embed pluginspage=\"http://www.adobe.com/svg/viewer/install/\" type=\"image/svg+xml\" src=\"#{data[:base]}/pages.svg\" style=\"margin-left: 50px; width: 1000px; height: #{40 + 20*data[:pages].size}px;\"></div>")
|
39
|
+
r.write("<h2>Most recent referrers broken down by pages</h2>")
|
40
|
+
r.write("<table><tr><th>Referer</th><th>Pages</th></tr>\n")
|
41
|
+
odd = true
|
42
|
+
data[:referers_to_pages].each do |k,v|
|
43
|
+
k = k[0..79] + "..." if k.length > 80
|
44
|
+
r.write("<tr class='#{odd ? 'odd':'even'}'><td class='name'>#{CGI.escapeHTML(k)}</td> <td><table class='breakdown'>")
|
45
|
+
total = v[:total]
|
46
|
+
if v.size > 2 # include :total
|
47
|
+
r.write("<tr><td class='count'>#{total}</td><td><strong>total</strong></td></tr>")
|
48
|
+
end
|
49
|
+
v.sort_by{|page,count| -count}.each do |page,count|
|
50
|
+
r.write("<tr><td class='count'>#{count}</td><td>#{page.to_s}</td></tr>") if page != :total
|
51
|
+
end
|
52
|
+
r.write("</table></td></tr>\n")
|
53
|
+
odd = !odd
|
54
|
+
end
|
55
|
+
r.write(FOOTER)
|
56
|
+
r.finish
|
57
|
+
end
|
58
|
+
|
59
|
+
def show_svg(src)
|
60
|
+
fields = []
|
61
|
+
data = []
|
62
|
+
src.each do |k,v|
|
63
|
+
if k != "-" # Excluding because of referers
|
64
|
+
k = k[0..79] + "..." if k.length > 80
|
65
|
+
fields << CGI.escapeHTML(k)
|
66
|
+
data << v
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
if fields.empty?
|
71
|
+
r = Rack::Response.new("No data")
|
72
|
+
return r.finish
|
73
|
+
end
|
74
|
+
|
75
|
+
graph = SVG::Graph::BarHorizontal.new(
|
76
|
+
:height => 40 + 20 * data.size,
|
77
|
+
:width => 1000,
|
78
|
+
:fields => fields.reverse
|
79
|
+
)
|
80
|
+
graph.add_data(:data => data.reverse)
|
81
|
+
graph.rotate_y_labels = false
|
82
|
+
graph.scale_integers = true
|
83
|
+
graph.key = false
|
84
|
+
r = Rack::Response.new
|
85
|
+
r["Content-Type"] = "image/svg+xml"
|
86
|
+
r.write(graph.burn)
|
87
|
+
r.finish
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
|
2
|
+
require 'sliding-stats/stats'
|
3
|
+
require 'sliding-stats/persist'
|
4
|
+
require 'cgi'
|
5
|
+
|
6
|
+
module SlidingStats
|
7
|
+
DEFAULT_WINDOW = 500
|
8
|
+
|
9
|
+
# Provides a "sliding window" over the stats. You provide
|
10
|
+
# a limit, and then feeds data into it. When the number of
|
11
|
+
# lines of data exceeds the limit, the oldest gets removed.
|
12
|
+
#
|
13
|
+
# The actual stats calculation is handled by the Stats class
|
14
|
+
#
|
15
|
+
# At any point you can extract stats from from the current
|
16
|
+
# window.
|
17
|
+
#
|
18
|
+
# The following options can be passed in the opts argument:
|
19
|
+
# * :limit => the number of stats lines to keep
|
20
|
+
# * :ignore => Requests where this matches *either* the referer *or* the request
|
21
|
+
# *or* the user agent will not be considered at all.
|
22
|
+
# * :request_methods => Array of HTTP methods to track. Defaults to :get
|
23
|
+
# as POST, PUT etc. on "normal" sites rarely happen
|
24
|
+
# on inbound referrals, and so we'd be likely to overcount
|
25
|
+
# access to a specific page
|
26
|
+
# * :exclude_[referers|pages] => Arrays that will be matched against
|
27
|
+
# REQUEST_URI and HTTP_REFERER to decide
|
28
|
+
# whether or not to exclude this request from
|
29
|
+
# the appropriate stats.
|
30
|
+
# * :rewrite_referer =>
|
31
|
+
# An Array of arrays consisting of regexps
|
32
|
+
# and a rewrite pattern to filter the
|
33
|
+
# HTTP_REFERER against
|
34
|
+
class Window
|
35
|
+
attr_reader :stats
|
36
|
+
|
37
|
+
def initialize app, opts = {}
|
38
|
+
@app = app
|
39
|
+
@limit = (opts[:limit] || DEFAULT_WINDOW).to_i
|
40
|
+
@exclude_referers = opts[:exclude_referers] || []
|
41
|
+
@rewrite_referers = opts[:rewrite_referers] || []
|
42
|
+
@request_methods = opts[:request_methods] || [:get]
|
43
|
+
@exclude_pages = opts[:exclude_pages] || []
|
44
|
+
@ignore = opts[:ignore] || []
|
45
|
+
@persist = opts[:persist]
|
46
|
+
|
47
|
+
@requests = []
|
48
|
+
if @persist.is_a?(Numeric)
|
49
|
+
@persist = SlidingStats::Persist.new(@persist)
|
50
|
+
@requests = @persist.load
|
51
|
+
end
|
52
|
+
@stats = Stats.new(@requests,@exclude_referers,@exclude_pages)
|
53
|
+
end
|
54
|
+
|
55
|
+
def call env
|
56
|
+
ref = env["HTTP_REFERER"] || "-"
|
57
|
+
req = env["REQUEST_URI"]
|
58
|
+
ua = env["HTTP_USER_AGENT"]
|
59
|
+
req_meth = env["REQUEST_METHOD"].downcase.to_sym
|
60
|
+
|
61
|
+
if @request_methods.include?(req_meth) && !@ignore.detect{|pat| ref =~ pat || req =~ pat || ua =~ pat}
|
62
|
+
newref = @rewrite_referers.inject(ref) { |nr,r| nr.gsub(r[0],r[1]) }
|
63
|
+
ref = CGI.unescape(newref) if newref != ref
|
64
|
+
|
65
|
+
stats = {
|
66
|
+
"HTTP_REFERER" => ref,
|
67
|
+
"REQUEST_URI" => req
|
68
|
+
}
|
69
|
+
@requests << stats
|
70
|
+
@stats.add(stats)
|
71
|
+
while @requests.size > @limit
|
72
|
+
@stats.sub(@requests.shift)
|
73
|
+
end
|
74
|
+
@persist.save(@requests) if @persist
|
75
|
+
end
|
76
|
+
|
77
|
+
env["slidingstats"] = self
|
78
|
+
@app.call(env)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
|
2
|
+
Gem::Specification.new do |s|
|
3
|
+
s.specification_version = 2 if s.respond_to? :specification_version=
|
4
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
5
|
+
|
6
|
+
s.name = 'sliding-stats'
|
7
|
+
s.version = '0.2.8'
|
8
|
+
s.date = '2011-10-14'
|
9
|
+
|
10
|
+
s.description = "Rack Middleware to provide a 'sliding view' over the last N requests to your web app"
|
11
|
+
s.summary = s.description
|
12
|
+
|
13
|
+
s.authors = ["vidarh"]
|
14
|
+
s.email = "vidar@hokstad.com"
|
15
|
+
|
16
|
+
# = MANIFEST =
|
17
|
+
s.files = %w[
|
18
|
+
README.rdoc
|
19
|
+
Rakefile
|
20
|
+
example/test.rb
|
21
|
+
features/stats.feature
|
22
|
+
features/step_definitions/stats_steps.rb
|
23
|
+
features/step_definitions/window_steps.rb
|
24
|
+
features/window.feature
|
25
|
+
lib/sliding-stats.rb
|
26
|
+
lib/sliding-stats/controller.rb
|
27
|
+
lib/sliding-stats/persist.rb
|
28
|
+
lib/sliding-stats/stats.rb
|
29
|
+
lib/sliding-stats/view.rb
|
30
|
+
lib/sliding-stats/window.rb
|
31
|
+
sliding-stats.gemspec
|
32
|
+
]
|
33
|
+
# = MANIFEST =
|
34
|
+
|
35
|
+
s.test_files = s.files.select {|path| path =~ /^test\/spec_.*\.rb/}
|
36
|
+
|
37
|
+
s.extra_rdoc_files = %w[]
|
38
|
+
s.add_dependency 'rack', '>= 0.9.1'
|
39
|
+
s.add_dependency 'svg_graph', '>= 0.7'
|
40
|
+
#s.add_development_dependency 'json', '>= 1.1'
|
41
|
+
|
42
|
+
s.has_rdoc = true
|
43
|
+
s.homepage = "http://www.hokstad.com/slidingstats"
|
44
|
+
s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "slidingstats", "--main", "README.rdoc"]
|
45
|
+
s.require_paths = %w[lib]
|
46
|
+
s.rubygems_version = '1.1.1'
|
47
|
+
end
|
metadata
ADDED
@@ -0,0 +1,113 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sliding-stats
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 7
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
- 8
|
10
|
+
version: 0.2.8
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- vidarh
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-10-14 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: rack
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 57
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
- 9
|
32
|
+
- 1
|
33
|
+
version: 0.9.1
|
34
|
+
type: :runtime
|
35
|
+
version_requirements: *id001
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: svg_graph
|
38
|
+
prerelease: false
|
39
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
41
|
+
requirements:
|
42
|
+
- - ">="
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
hash: 5
|
45
|
+
segments:
|
46
|
+
- 0
|
47
|
+
- 7
|
48
|
+
version: "0.7"
|
49
|
+
type: :runtime
|
50
|
+
version_requirements: *id002
|
51
|
+
description: Rack Middleware to provide a 'sliding view' over the last N requests to your web app
|
52
|
+
email: vidar@hokstad.com
|
53
|
+
executables: []
|
54
|
+
|
55
|
+
extensions: []
|
56
|
+
|
57
|
+
extra_rdoc_files: []
|
58
|
+
|
59
|
+
files:
|
60
|
+
- README.rdoc
|
61
|
+
- Rakefile
|
62
|
+
- example/test.rb
|
63
|
+
- features/stats.feature
|
64
|
+
- features/step_definitions/stats_steps.rb
|
65
|
+
- features/step_definitions/window_steps.rb
|
66
|
+
- features/window.feature
|
67
|
+
- lib/sliding-stats.rb
|
68
|
+
- lib/sliding-stats/controller.rb
|
69
|
+
- lib/sliding-stats/persist.rb
|
70
|
+
- lib/sliding-stats/stats.rb
|
71
|
+
- lib/sliding-stats/view.rb
|
72
|
+
- lib/sliding-stats/window.rb
|
73
|
+
- sliding-stats.gemspec
|
74
|
+
homepage: http://www.hokstad.com/slidingstats
|
75
|
+
licenses: []
|
76
|
+
|
77
|
+
post_install_message:
|
78
|
+
rdoc_options:
|
79
|
+
- --line-numbers
|
80
|
+
- --inline-source
|
81
|
+
- --title
|
82
|
+
- slidingstats
|
83
|
+
- --main
|
84
|
+
- README.rdoc
|
85
|
+
require_paths:
|
86
|
+
- lib
|
87
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
88
|
+
none: false
|
89
|
+
requirements:
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
hash: 3
|
93
|
+
segments:
|
94
|
+
- 0
|
95
|
+
version: "0"
|
96
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ">="
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
hash: 3
|
102
|
+
segments:
|
103
|
+
- 0
|
104
|
+
version: "0"
|
105
|
+
requirements: []
|
106
|
+
|
107
|
+
rubyforge_project:
|
108
|
+
rubygems_version: 1.8.6
|
109
|
+
signing_key:
|
110
|
+
specification_version: 2
|
111
|
+
summary: Rack Middleware to provide a 'sliding view' over the last N requests to your web app
|
112
|
+
test_files: []
|
113
|
+
|