metry 1.0.0

data/TODO

@@ -0,0 +1,10 @@
+More unit tests
+
+Add support for non-pageview event tracking
+  Via in-process call
+  Via web request
+
+Add support for MongoDB
+Add support for Tokyo Tyrant
+
+Metry visitors can be spoofed - fix it

data/example/example.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'sinatra'
+
+require File.dirname(__FILE__) + '/../lib/metry'
+
+configure do
+  Metry.init Metry::Tokyo.new("tracking")
+  #Metry.init Metry::Memory.new
+  use Metry::Rack::Tracking
+end
+
+get '/' do
+  "Root"
+end
+
+get '/subpage' do
+  "Sub page"
+end
+
+get '/extra' do
+  request.env["metry.event"]["extra"] = params[:track]
+end

data/features/app_tracking.feature

@@ -0,0 +1,11 @@
+Feature: Application Tracking
+  In order to tie my tracking in with interesting application-level stats
+  As an application developer
+  I want to track additional facets of data from my application
+  
+  Scenario: Track an additional facet
+    Given an empty tracking database
+    When I view "/extra?track=stuff"
+    Then there should be a tracking event "1":
+      | key   | value |
+      | extra | stuff |

data/features/basic_tracking.feature

@@ -0,0 +1,88 @@
+Feature: Access Tracking
+  In order to how users are using my site
+  As an web application creator
+  I want to have all accesses to my application tracked
+  
+  Background:
+    Given an empty tracking database
+    And I am a new visitor
+  
+  Scenario: Single gets are tracked
+    When I view "/"
+    Then there should be 1 tracking event
+    
+  Scenario: Multiple gets are tracked
+    When I view "/"
+    And I view "/"
+    And I view "/"
+    Then there should be 3 tracking events
+    
+  Scenario: Basic request data is tracked
+    When I view "/"
+    And I view "/subpage"
+    Then there should be a tracking event "1":
+      | key  | value    |
+      | path | /        |
+      | time | _exists_ |
+    And there should be a tracking event "2":
+      | key  | value    |
+      | path | /subpage |
+      | time | _exists_ |
+    
+  Scenario: New visitor is tracked
+    When I view "/"
+    And I view "/subpage"
+    Then there should be a tracking event "1":
+      | key     | value |
+      | visitor | 1     |
+    And there should be a tracking event "2":
+      | key     | value |
+      | visitor | 1     |
+
+  Scenario: Two visitors are tracked
+    Given I view "/"
+    When I am a new visitor
+    Given I view "/"
+    Then there should be a tracking event "1":
+      | key     | value |
+      | visitor | 1     |
+    And there should be a tracking event "2":
+      | key     | value |
+      | visitor | 2     |
+      
+  Scenario: All facets should be tracked
+    When I view "/"
+    Then there should be a tracking event "1":
+      | key  | value       |
+      | path | /           |
+      | time | _exists_    |
+      | ip   | 127.0.0.1   |
+      | host | example.org |
+
+  Scenario: path should include query string
+    When I view "/?here=there"
+    Then there should be a tracking event "1":
+      | key          | value        |
+      | path | /?here=there |
+      
+  Scenario: Should track status codes
+    When I view "/"
+    And I view "/missing"
+    Then there should be a tracking event "1":
+      | key    | value |
+      | status | 200   |
+    Then there should be a tracking event "2":
+      | key    | value |
+      | status | 404   |
+
+  Scenario: Should track method
+    When I view "/"
+    And I post to "/post":
+      | key   | value |
+      | bogus | bogus |
+    Then there should be a tracking event "1":
+      | key    | value |
+      | method | GET   |
+    Then there should be a tracking event "2":
+      | key    | value |
+      | method | POST  |

data/features/sample_application.feature

@@ -0,0 +1,12 @@
+Feature: Basic Sinatra application works
+  In order to test rack middleware
+  As a middleware creator
+  I want an example application that I can test against
+  
+  Scenario: Get root
+    When I view "/"
+    Then I should see "Root"
+
+  Scenario: Get subpage
+    When I view "/subpage"
+    Then I should see "Sub page"

data/features/step_definitions/tracking.rb

@@ -0,0 +1,35 @@
+Given /^an empty tracking database$/ do
+  Metry.current.clear
+end
+
+Then /^there should be (\d+) tracking events?$/ do |event_count|
+  assert_equal(event_count.to_i, Metry.current.event_count)
+end
+
+When /^there should be a tracking event "(\d+)":$/ do |id, table|
+  event = Metry.current[id]
+  assert event, "Unable to lookup event #{id}."
+  table.hashes.each do |hash|
+    expected = hash["value"]
+    case expected
+    when "_exists_"
+      assert event[hash["key"]], "Key #{hash["key"]} does not exist."
+    else
+      assert_equal expected, event[hash["key"]], "Key #{hash["key"]} does not match."
+    end
+  end
+end
+
+Then /^there should be a visitor "([^\"]*)":$/ do |id, table|
+  visitor = Metry.current.visitor(id)
+  assert visitor, "Unable to lookup visitor #{id}."
+  table.hashes.each do |hash|
+    expected = hash["value"]
+    case expected
+    when "_exists_"
+      assert visitor[hash["key"]], "Key #{hash["key"]} does not exist."
+    else
+      assert_equal expected, visitor[hash["key"]], "Key #{hash["key"]} does not match."
+    end
+  end
+end

data/features/step_definitions/web.rb

@@ -0,0 +1,31 @@
+Given /^I am a new visitor$/ do
+  clear_cookies
+end
+
+When /^I view "([^\"]*)"$/ do |path|
+  visit(path)
+end
+
+When /^I (post|put) to "([^\"]*)":$/ do |meth, path, params|
+  send(meth, path, params.hashes.inject({}){|h,e| h[e["key"]] = e["value"]; h})
+end
+
+Then /^I should see "([^\"]*)"$/ do |string|
+  assert_match(string, last_response.body)
+end
+
+When /^I follow "([^\"]*)"$/ do |link|
+  click_link(link)
+end
+
+When /^I fill in "([^\"]*)" with "([^\"]*)"$/ do |field, value|
+  fill_in(field, :with => value)
+end
+
+When /^I press "([^\"]*)"$/ do |button|
+  click_button(button)
+end
+
+Then /^I should be on "([^\"]*)"$/ do |path|
+  assert_equal path, last_response.path
+end

data/features/support/env.rb

@@ -0,0 +1,22 @@
+require 'test/unit/assertions'
+World(Test::Unit::Assertions)
+
+require 'rack/test'
+
+$: << File.dirname(__FILE__) + "/../../vendor/webrat/lib"
+require 'webrat'
+require 'logger'
+Webrat.configure do |config|
+  config.mode = :rack_test
+end
+
+World(Rack::Test::Methods)
+World(Webrat::Methods)
+World(Webrat::Matchers)
+
+require File.dirname(__FILE__) + '/../../example/example.rb'
+def app
+  Rack::Builder.new do
+    run Sinatra::Application
+  end
+end

data/lib/metry.rb

@@ -0,0 +1,19 @@
+$: << File.dirname(__FILE__)
+$: << File.dirname(__FILE__) + '/../vendor/rufus-tokyo/lib'
+
+require 'metry/rack/tracking'
+require 'metry/memory'
+require 'metry/tokyo'
+require 'metry/experiment'
+
+module Metry
+  VERSION = '1.0.0'
+  
+  def self.init(storage)
+    @storage = storage
+  end
+  
+  def self.current
+    @storage
+  end
+end

data/lib/metry/experiment.rb

@@ -0,0 +1,25 @@
+module Metry
+  class Experiment
+    METHODS = {
+      "rand" => proc{|list, visitor| list.sort_by{rand}.first},
+      "mod_visitor" => proc{|list, visitor| list[(visitor.to_i-1)%list.size]},
+    }
+
+    def initialize(name, event)
+      @key = "experiment.#{name}"
+      @event = event
+      @visitor = event["visitor"]
+      @storage = Metry.current
+    end
+    
+    def choose(options, method=nil)
+      visitor_hash = @storage.visitor(@visitor)
+      unless choice = visitor_hash[@key]
+        choice = METHODS[method || "rand"][options.keys, @visitor]
+        @storage.save_visitor(@visitor, visitor_hash.merge(@key => choice))
+      end
+      @event[@key] = choice
+      options[choice]
+    end
+  end
+end

data/lib/metry/memory.rb

@@ -0,0 +1,27 @@
+module Metry
+  class Memory
+    def initialize
+      clear
+    end
+    
+    def <<(event)
+      @hash[:events][(@hash[:event] += 1).to_s] = event
+    end
+    
+    def size
+      @hash[:events].size
+    end
+    
+    def [](id)
+      @hash[:events][id]
+    end
+    
+    def next_visitor
+      @hash[:visitor] += 1
+    end
+    
+    def clear
+      @hash = {:events => {}, :visitor => 0, :event => 0}
+    end
+  end
+end

data/lib/metry/rack/tracking.rb

@@ -0,0 +1,51 @@
+module Metry
+  module Rack
+    class Tracking
+      COOKIE = "_metry"
+
+      def initialize(app)
+        @app = app
+        @storage = Metry.current
+      end
+    
+      def call(env)
+        request = ::Rack::Request.new(env)
+        visitor = find_visitor(request)
+        env["metry.event"] = event = build_event(visitor, request, env)
+
+        status, headers, body = @app.call(env)
+
+        event["status"] = status.to_s
+        @storage << event
+
+        response = ::Rack::Response.new(body, status, headers)
+        save_visitor(response, visitor)
+
+        response.to_a
+      end
+    
+      def build_event(visitor, request, env)
+        { "event" => "pageview",
+          "path" => request.fullpath,
+          "time" => Time.now.to_f.to_s.sub(/\.(\d+)/){|m| [$1[0...5].ljust(5, "0")]},
+          "visitor" => visitor,
+          "ip" => request.ip,
+          "host" => request.host,
+          "method" => request.request_method,
+          "referrer" => env["HTTP_REFERER"],
+          "user_agent" => env["HTTP_USER_AGENT"] }
+      end
+    
+      def find_visitor(request)
+        (request.cookies[COOKIE] || @storage.next_visitor.to_s)
+      end
+    
+      def save_visitor(response, visitor)
+        response.set_cookie(COOKIE,
+          :value => visitor,
+          :expires => (Time.now+(60*60*24*365*20)),
+          :path => '/')
+      end
+    end
+  end
+end

data/lib/metry/tokyo.rb

@@ -0,0 +1,153 @@
+require 'rubygems'
+require 'rufus/tokyo'
+require 'rufus/edo'
+
+require 'fileutils'
+
+module Metry
+  class Tokyo
+    EVENT_PREFIX = "event_"
+    VISITOR_PREFIX = "visitor_"
+    META_PREFIX = "meta_"
+    
+    def initialize(file, interface="Edo")
+      FileUtils.mkdir_p(::File.dirname(file))
+      @file = "#{file}.tdb"
+      @store = Rufus.const_get(interface)::Table
+    end
+    
+    def next_visitor
+      write do |storage|
+        storage.incr("#{META_PREFIX}visitor")
+      end
+    end
+    
+    def visitor(id)
+      read do |storage|
+        (storage[visitor_id(id)] || {})
+      end
+    end
+    
+    def save_visitor(id, hash)
+      write do |storage|
+        storage[visitor_id(id)] = hash
+      end
+    end
+    
+    def <<(event)
+      write do |storage|
+        storage[next_event_id(storage)] = event.inject({}){|a,(k,v)| a[k] = v.to_s; a}
+      end
+    end
+    
+    def [](id)
+      read do |storage|
+        storage[event_id(id)]
+      end
+    end
+    
+    def event_count
+      read do |storage|
+        storage.keys(:prefix => EVENT_PREFIX).size
+      end
+    end
+    
+    def last_events(count=1)
+      read do |storage|
+        storage.query do |q|
+          q.add '', :starts_with, EVENT_PREFIX
+          q.order_by "time", :numdesc
+          q.limit count
+        end.to_a
+      end
+    end
+    
+    def all_events
+      read do |storage|
+        storage.query do |q|
+          q.add '', :starts_with, EVENT_PREFIX
+        end.to_a
+      end
+    end
+    
+    def clear
+      write do |storage|
+        storage.clear
+      end
+    end
+    
+    private
+    
+    def next_event_id(storage)
+      event_id(storage.incr("#{META_PREFIX}event"))
+    end
+    
+    def access(mode)
+      storage = @store.new(@file, :mode => mode)
+      begin
+        result = yield(storage)
+      ensure
+        storage.close
+      end
+      result
+    end
+    
+    def read(&block)
+      access("rc", &block)
+    end
+    
+    def write(&block)
+      access("wc", &block)
+    end
+    
+    def visitor_id(id)
+      "#{VISITOR_PREFIX}#{id}"
+    end
+    
+    def event_id(id)
+      "#{EVENT_PREFIX}#{id}"
+    end
+  end
+end
+
+module Rufus::Tokyo
+  class Table
+    # Increments the value stored under the given key with the given increment
+    # (defaults to 1 (integer)).
+    #
+    # Accepts an integer or a double value.
+    #
+    def incr (key, inc=1)
+
+      v = inc.is_a?(Fixnum) ?
+        lib.addint(@db, key, Rufus::Tokyo.blen(key), inc) :
+        lib.adddouble(@db, key, Rufus::Tokyo.blen(key), inc)
+
+      raise(TokyoError.new(
+        "incr failed, there is probably already a string value set " +
+        "for the key '#{key}'"
+      )) if v == Rufus::Tokyo::INT_MIN || (v.respond_to?(:nan?) && v.nan?)
+
+      v
+    end
+  end
+end
+
+module Rufus::Edo
+  class Table
+    # Increments the value stored under the given key with the given increment
+    # (defaults to 1 (integer)).
+    #
+    def incr (key, val=1)
+
+      v = val.is_a?(Fixnum) ? @db.addint(key, val) : @db.adddouble(key, val)
+
+      raise(EdoError.new(
+        "incr failed, there is probably already a string value set " +
+        "for the key '#{key}'"
+      )) unless v
+
+      v
+    end
+  end
+end