the-official-groupme-ab-testing-solution 0.0.1

data/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+
+gem "redis"
+
+group :test do
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    redis (2.2.0)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.1)
+    rspec-expectations (2.5.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  redis
+  rspec

data/README.md

@@ -0,0 +1,77 @@
+# The Official GroupMe AB Testing Solution
+
+Simple AB testing with Redis.
+
+## USAGE
+
+Define an AB test:
+
+    AB.define :new_twitter, "Show new Twitter or old Twitter to logged in users" do
+      alternative(:new) { "new-twitter.erb" }
+      alternative(:old) { "old-twitter.erb" }
+    end
+
+Currently, identity of experiment participants is determined by an integer you
+pass when the participant views the experiment. Use `AB.test` to show always show
+a participant a consistent experience.
+
+    class TwitterController
+      def show
+        render :template => AB.test(:new_twitter, current_user.id)
+      end
+    end
+
+If you don't have a current user, you can set a cookie or something:
+
+    class HomepageController
+      def index
+        identifier = session[:ab_identifier] ||= rand(100)
+        render :template => AB.test(:home_page, identifier)
+      end
+    end
+
+To track conversions, use `AB.track`, passing the participant's identifier:
+
+    class TwitterController
+      def spend_dollahs
+        AB.track(:new_twitter, current_user.id)
+      end
+    end
+
+or with a cookie:
+
+    class HomepageController
+      def signup
+        AB.test(:home_page, session[:ab_identifier])
+      end
+    end
+
+To get at your results, you can get a test, and call `results`.
+
+    AB.get(:home_page).results
+
+## Testing
+
+In tests, if you want to guarantee a certain alternative will be shown,
+you can do so by specifying it with `use`
+
+    it "should say 'NEW TWITTER' showing new Twitter" do
+      AB.test(:new_twitter).use(:new) do
+        get "/twitter/show"
+        page.should have_content("NEW TWITTER")
+      end
+    end
+
+## Redis
+
+This gem relies on the redis gem, it looks for a connection by just
+booting up a Redis object with `Redis.new`. If you want to connect to
+an alternative redis, you can assign it to `AB.redis`.
+
+    AB.redis = Redis.new('redis://something-weird')
+
+### TODO
+
+* More interesting statistics, including relevance information
+* Pretty printing of stats to the command line
+* Pretty web interface

data/Rakefile

@@ -0,0 +1,8 @@
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = "--color"
+  t.pattern = FileList['spec/**/*_spec.rb']
+end
+
+task :default => :spec

data/lib/ab.rb

@@ -0,0 +1,17 @@
+$LOAD_PATH << File.dirname(__FILE__)
+
+require "ab/test"
+require "ab/interface"
+require "ab/alternative"
+
+module AB
+  VERSION = "0.0.1"
+
+  def self.redis=(redis)
+    @redis = redis
+  end
+
+  def self.redis
+    @redis ||= Redis.new
+  end
+end

data/lib/ab/alternative.rb

@@ -0,0 +1,66 @@
+module AB
+  class Alternative
+    attr_reader :name
+    attr_accessor :range
+
+    def initialize(test, name, options = {}, &value_proc)
+      @test, @name, @options, @value_proc = test, name, options, value_proc
+      @test.update_percentage_range(self) if percent
+    end
+
+    def percent
+      @options[:percent]
+    end
+
+    def value
+      @value ||= @value_proc.call
+    end
+
+    def views_key
+      @views_key ||= "ab:#{@test.name}:#{@name}"
+    end
+
+    def view!(identity)
+      return if identity.nil?
+      return if redis.zscore(views_key, identity)
+      score = Time.now.to_i
+      redis.zadd(views_key, score, identity)
+    end
+
+    def views
+      redis.zcard(views_key)
+    end
+
+    def conversions_key
+      @conversions_key ||= views_key + ":conversions"
+    end
+
+    def track_conversion!(identity)
+      return if identity.nil?
+      return if redis.zscore(conversions_key, identity)
+      return unless redis.zscore(views_key, identity)
+      score = Time.now.to_i
+      redis.zadd(conversions_key, score, identity)
+    end
+
+    def conversions
+      redis.zcard(conversions_key)
+    end
+
+    def results
+      {
+        :name => name,
+        :value => value,
+        :views => views,
+        :conversions => conversions,
+        :percent => percent
+      }
+    end
+
+    private
+
+    def redis
+      AB.redis
+    end
+  end
+end

data/lib/ab/interface.rb

@@ -0,0 +1,42 @@
+module AB
+  class << self
+    def define(name, description, &block)
+      raise ArgumentError, "#{name.inspect} is already an AB test" if tests.has_key?(name)
+      t = AB::Test.new(name, description)
+      t.instance_eval(&block) if block_given?
+      t.verify_percentages
+      tests[name] = t
+    end
+
+    def get(name)
+      tests[name]
+    end
+
+    def track(name, identity)
+      test = get(name)
+      alternative = test.get_alternative(identity)
+      begin
+        alternative.track_conversion!(identity)
+      rescue Timeout::Error
+        # Don't track data if redis is down
+      end
+    end
+
+    def test(name, identity)
+      test = get(name)
+      alternative = test.get_alternative(identity)
+      begin
+        alternative.view!(identity)
+      rescue Timeout::Error
+        # Don't track data if redis is down
+      end
+      alternative.value
+    end
+
+    private
+
+    def tests
+      @tests ||= {}
+    end
+  end
+end

data/lib/ab/test.rb

@@ -0,0 +1,54 @@
+module AB
+  class Test
+    attr_reader :name, :description, :alternatives
+
+    def initialize(name, description)
+      @name, @description, @alternatives, @last_percent= name, description, [], 0
+    end
+
+    def alternative(name, options={}, &value_proc)
+      alternative = AB::Alternative.new(self, name, options, &value_proc)
+      alternatives << alternative
+      alternative
+    end
+
+    def use(name)
+      begin
+        @using = alternatives.detect { |alternative| alternative.name == name }
+        yield
+      ensure
+        @using = nil
+      end
+    end
+
+    def get_alternative(identity)
+      return @using if @using
+
+      if identity.nil?
+        return alternatives.first
+      end
+
+      if alternatives.any?(&:percent)
+        alternatives.detect { |alternative| alternative.range.include?(identity % 100) }
+      else
+        alternatives[identity % alternatives.size]
+      end
+    end
+
+    def update_percentage_range(alternative)
+      percent = alternative.percent - 1
+      alternative.range = @last_percent..(@last_percent + percent)
+      @last_percent = alternative.range.last + 1
+    end
+
+    def verify_percentages
+      return unless alternatives.any?(&:percent)
+      raise ArgumentError.new("Some alternatives are missing percentages") unless alternatives.all?(&:percent)
+      raise ArgumentError.new("Percentages must add up to 100") unless @last_percent == 100
+    end
+
+    def results
+      alternatives.map(&:results)
+    end
+  end
+end

data/spec/ab/alternative_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+describe AB::Alternative do
+  before do
+    @test = AB::Test.new(:home_page, "Red or blue home page")
+    AB.redis.flushall
+  end
+
+  describe "#value" do
+    it "returns the result of the value_proc" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      alternative.value.should == "red.erb"
+    end
+  end
+
+  describe "views_key" do
+    it "returns the experiments namespace with the test name and alternative name" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      alternative.views_key.should == "ab:home_page:red"
+    end
+  end
+
+  describe "view!" do
+    it "tracks the view (once per identity)" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+
+      running {
+        alternative.view! 123
+      }.should change { alternative.views }
+
+      running {
+        alternative.view! 123
+      }.should_not change { alternative.views }
+    end
+
+    it "does not update timestamp" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      AB.redis.zadd(alternative.views_key, 1000, 123)
+      alternative.view! 123
+      AB.redis.zscore(alternative.views_key, 123).should == "1000"
+    end
+
+    it "does not track the view if the identity is nil" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      running {
+        alternative.view! nil
+      }.should_not change { AB.redis.dbsize }
+    end
+  end
+
+  describe "views" do
+    it "returns the number of views tracked" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      alternative.views.should == 0
+      alternative.view! 123
+      alternative.views.should == 1
+      alternative.view! 789
+      alternative.views.should == 2
+      alternative.view! 123
+      alternative.views.should == 2
+    end
+  end
+
+  describe "conversions_key" do
+    it "returns views_key with :conversions suffix" do
+      alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+      alternative.conversions_key.should == "ab:home_page:red:conversions"
+    end
+  end
+
+  describe "track_conversion!" do
+    before do
+      @alternative = AB::Alternative.new(@test, :red) { "red.erb" }
+    end
+
+    it "tracks the conversion (once per identity)" do
+      @alternative.view! 123
+
+      running {
+        @alternative.track_conversion! 123
+      }.should change { @alternative.conversions }.by(1)
+
+      running {
+        @alternative.track_conversion! 123
+      }.should_not change { @alternative.conversions }
+    end
+
+    it "does not track conversion if identity did not view alternative" do
+      running {
+        @alternative.track_conversion! 123
+      }.should_not change { @alternative.conversions }
+    end
+
+    it "does not track the conversion if the identity is nil" do
+      running {
+        @alternative.track_conversion! nil
+      }.should_not change { AB.redis.dbsize }
+    end
+  end
+end

data/spec/ab/interface_spec.rb

@@ -0,0 +1,114 @@
+require "spec/spec_helper"
+
+describe AB do
+  after do
+    AB.send(:tests).delete(:home_page)
+    AB.redis.flushall
+  end
+
+  describe ".define" do
+    it "defines an AB test" do
+      AB.define :home_page, "Big red button or big blue button versions" do
+        alternative(:red) { "red.erb" }
+        alternative(:blue) { "blue.erb" }
+      end
+
+      test = AB.get(:home_page)
+      test.name.should == :home_page
+      test.description.should == "Big red button or big blue button versions"
+      test.should have(2).alternatives
+      red_alternative = test.alternatives.first
+      red_alternative.name.should == :red
+      red_alternative.value.should == "red.erb"
+    end
+
+    it "raises when you try to define the same test twice" do
+      AB.define(:home_page, "Big red button")
+      running {
+        AB.define(:home_page, "Other big red button")
+      }.should raise_error(ArgumentError)
+    end
+
+    it "can set a percentage for alternatives" do
+      AB.define :home_page, "Red button or blue button" do
+        alternative(:red, :percent => 10) { "red.erb" }
+        alternative(:red, :percent => 90) { "blue.erb" }
+      end
+
+      0.upto(9) { |i| AB.test(:home_page, i).should == "red.erb" }
+      10.upto(99) { |i| AB.test(:home_page, i).should == "blue.erb" }
+    end
+
+    it "explodes if percentage sum does not equal 100" do
+      running {
+        AB.define :home_page, "Red button or blue button" do
+          alternative(:red, :percent => 10) { "red.erb" }
+          alternative(:red, :percent => 80) { "blue.erb" }
+        end
+      }.should raise_error(ArgumentError)
+    end
+
+    it "explodes if percentages are only set on some of the alternatives" do
+      running {
+        AB.define :home_page, "Red button or blue button" do
+          alternative(:red, :percent => 10) { "red.erb" }
+          alternative(:red) { "blue.erb" }
+        end
+      }.should raise_error(ArgumentError)
+    end
+  end
+
+  describe ".test" do
+    before do
+      @test = AB.define :home_page, "Red or Blue home page" do
+        alternative(:red) { "red.erb" }
+        alternative(:blue) { "blue.erb" }
+      end
+    end
+
+    it "returns an alternative value" do
+      value = AB.test(:home_page, 123)
+      ["red.erb", "blue.erb"].should include(value)
+    end
+
+    it "increases the views for the alternative" do
+      alternative = @test.get_alternative(123)
+      running {
+        AB.test(:home_page, 123)
+      }.should change { alternative.views }.by(1)
+    end
+
+    it "rescues Redis failures and still returns a value" do
+      AB.redis.should_receive(:zadd).and_raise(Timeout::Error.new("time's up!"))
+      running {
+        value = AB.test(:home_page, 123)
+        ["red.erb", "blue.erb"].should include(value)
+      }.should_not raise_error
+    end
+  end
+
+  describe ".track" do
+    before do
+      @test = AB.define :home_page, "Red or blue home page" do
+        alternative(:red) { "red.erb" }
+        alternative(:blue) { "blue.erb" }
+      end
+
+      @alternative = @test.get_alternative(123)
+      @alternative.view! 123
+    end
+
+    it "increases the conversions for the alternative" do
+      running {
+        AB.track :home_page, 123
+      }.should change { @alternative.conversions }.by(1)
+    end
+
+    it "rescues Redis timeout failures" do
+      AB.redis.should_receive(:zscore).and_raise(Timeout::Error.new("time's up!"))
+      running {
+        AB.track :home_page, 123
+      }.should_not raise_error
+    end
+  end
+end

data/spec/ab/test_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+describe AB::Test do
+  describe "#get_alternative" do
+    before do
+      @test = AB::Test.new(:home_page, "Red or blue home page")
+      @red = @test.alternative(:red) { "red.erb" }
+      @blue = @test.alternative(:blue) { "blue.erb" }
+    end
+
+    it "returns the value of an alternative based on the identity (50/50)" do
+      @test.get_alternative(0).should == @red
+      @test.get_alternative(1).should == @blue
+    end
+
+    it "returns the value of an alternative based on the identity (33/33/33)" do
+      @green = @test.alternative(:green) { "green.erb" } # add an additional alternative
+      @test.get_alternative(0).should == @red
+      @test.get_alternative(1).should == @blue
+      @test.get_alternative(2).should == @green
+    end
+
+    it "returns the first alternative when identity is nil" do
+      @test.get_alternative(nil).should == @red
+    end
+  end
+
+  describe "#verify_percentages" do
+    before do
+      @test = AB::Test.new(:home_page, "Red or blue home page")
+    end
+
+    it "raises ArgumentError when some alternatives do not have percentages and some do" do
+      @test.alternative(:red, :percent => 10) { "red.erb" }
+      @test.alternative(:blue) { "blue.erb" }
+      running {
+        @test.verify_percentages
+      }.should raise_error(ArgumentError)
+    end
+
+    it "raises ArgumentError when percentages do not add up to 100" do
+      @test.alternative(:red, :percent => 10) { "red.erb" }
+      @test.alternative(:blue, :percent => 10) { "blue.erb" }
+      running {
+        @test.verify_percentages
+      }.should raise_error(ArgumentError)
+    end
+  end
+
+  describe "#results" do
+    before do
+      @test = AB::Test.new(:home_page, "Red or blue home page")
+      @red = @test.alternative(:red) { "red.erb" }
+      @blue = @test.alternative(:blue) { "blue.erb" }
+    end
+
+    it "returns list of alternatives with stats" do
+      @red.view! 123
+      @red.track_conversion! 123
+      @test.results.should == [
+        { :name => :red, :value => "red.erb", :views => 1, :conversions => 1, :percent => nil },
+        { :name => :blue, :value => "blue.erb", :views => 0, :conversions => 0, :percent => nil }
+      ]
+    end
+  end
+
+  describe "#use" do
+    it "always returns the alternative passed in the block" do
+      test = AB::Test.new(:home_page, "Red or blue home page")
+      red = test.alternative(:red) { "red.erb" }
+      blue = test.alternative(:blue) { "blue.erb" }
+
+      test.use :red do
+        test.get_alternative(0).should == red
+        test.get_alternative(1).should == red
+      end
+
+      test.get_alternative(0).should == red
+      test.get_alternative(1).should == blue
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require "redis"
+require 'yaml'
+
+require File.join(File.dirname(__FILE__), *%w[.. lib ab])
+
+alias running proc

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+name: the-official-groupme-ab-testing-solution
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Dave Yeu and Pat Nakajima
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-28 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: 
+email: pat@groupme.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- lib/ab.rb
+- lib/ab/alternative.rb
+- lib/ab/interface.rb
+- lib/ab/test.rb
+- spec/ab/alternative_spec.rb
+- spec/ab/interface_spec.rb
+- spec/ab/test_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/groupme/the-official-groupme-ab-testing-solution
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: The Official GroupMe AB Testing Solution
+test_files: []
+