bananasplit 2.0.3

data/lib/bananasplit/alternative.rb

@@ -0,0 +1,22 @@
+require "active_record"
+require_relative "conversion_rate"
+class BananaSplit::Alternative < ActiveRecord::Base
+  include BananaSplit::ConversionRate
+
+  belongs_to :experiment, :class_name => "BananaSplit::Experiment"
+  attr_accessible :content, :weight, :lookup
+  serialize :content
+
+  def self.calculate_lookup(test_name, alternative_name)
+    Digest::MD5.hexdigest(BananaSplit.salt + test_name + alternative_name.to_s)
+  end
+
+  def self.score_conversion(test_name, viewed_alternative)
+    self.update_all("conversions = conversions + 1", :lookup => self.calculate_lookup(test_name, viewed_alternative))
+  end
+
+  def self.score_participation(test_name, viewed_alternative)
+    self.update_all("participants = participants + 1", :lookup => self.calculate_lookup(test_name, viewed_alternative))
+  end
+
+end

data/lib/bananasplit/controller/dashboard.rb

@@ -0,0 +1,25 @@
+require 'action_controller'
+class BananaSplit
+  module Controller
+    module Dashboard
+      ActionController::Base.prepend_view_path File.join(File.dirname(__FILE__), "../views")
+
+      def index
+        @experiments = BananaSplit::Experiment.all
+        render :template => 'dashboard/index'
+      end
+
+      def end_experiment
+        @alternative = BananaSplit::Alternative.find(params[:id])
+        @experiment  = BananaSplit::Experiment.find(@alternative.experiment_id)
+        if (@experiment.status != "Completed")
+          @experiment.end_experiment!(@alternative.content)
+          flash[:notice] = "Experiment marked as ended.  All users will now see the chosen alternative."
+        else
+          flash[:notice] = "This experiment is already ended."
+        end
+        redirect_to :action => "index"
+      end
+    end
+  end
+end

data/lib/bananasplit/conversion_rate.rb

@@ -0,0 +1,9 @@
+module BananaSplit::ConversionRate
+  def conversion_rate
+    1.0 * conversions / participants
+  end
+
+  def pretty_conversion_rate
+    sprintf("%4.2f%%", conversion_rate * 100)
+  end
+end

data/lib/bananasplit/experiment.rb

@@ -0,0 +1,103 @@
+require_relative "statistics"
+require_relative "conversion_rate"
+class BananaSplit::Experiment < ActiveRecord::Base
+  include BananaSplit::Statistics
+  include BananaSplit::ConversionRate
+
+  has_many :alternatives, :dependent => :destroy, :class_name => "BananaSplit::Alternative"
+  validates_uniqueness_of :test_name
+  attr_accessible :test_name
+  before_destroy :cleanup_cache
+
+  def cache_keys
+  ["BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_"),
+    "BananaSplit::Experiment::#{test_name}::alternatives".gsub(" ","_"),
+    "BananaSplit::Experiment::short_circuit(#{test_name})".gsub(" ", "_")
+  ]
+  end
+
+  def cleanup_cache
+    cache_keys.each do |key|
+      BananaSplit.cache.delete key
+    end
+    true
+  end
+
+  def participants
+    alternatives.sum("participants")
+  end
+
+  def conversions
+    alternatives.sum("conversions")
+  end
+
+  def best_alternative
+    alternatives.max do |a,b|
+      a.conversion_rate <=> b.conversion_rate
+    end
+  end
+
+  def self.exists?(test_name)
+    cache_key = "BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_")
+    ret = BananaSplit.cache.fetch(cache_key) do
+      count = BananaSplit::Experiment.count(:conditions => {:test_name => test_name})
+      count > 0 ? count : nil
+    end
+    (!ret.nil?)
+  end
+
+  def self.alternatives_for_test(test_name)
+    cache_key = "BananaSplit::#{test_name}::alternatives".gsub(" ","_")
+    BananaSplit.cache.fetch(cache_key) do
+      experiment = BananaSplit::Experiment.find_by_test_name(test_name)
+      alternatives_array = BananaSplit.cache.fetch(cache_key) do
+        tmp_array = experiment.alternatives.map do |alt|
+          [alt.content, alt.weight]
+        end
+        tmp_hash = tmp_array.inject({}) {|hash, couplet| hash[couplet[0]] = couplet[1]; hash}
+        BananaSplit.parse_alternatives(tmp_hash)
+      end
+      alternatives_array
+    end
+  end
+
+  def self.start_experiment!(test_name, alternatives_array, conversion_name = nil)
+    conversion_name ||= test_name
+    conversion_name.gsub!(" ", "_")
+    cloned_alternatives_array = alternatives_array.clone
+    ActiveRecord::Base.transaction do
+      experiment = BananaSplit::Experiment.find_or_create_by_test_name(test_name)
+      experiment.alternatives.destroy_all  #Blows away alternatives for pre-existing experiments.
+      while (cloned_alternatives_array.size > 0)
+        alt = cloned_alternatives_array[0]
+        weight = cloned_alternatives_array.size - (cloned_alternatives_array - [alt]).size
+        experiment.alternatives.build(:content => alt, :weight => weight,
+          :lookup => BananaSplit::Alternative.calculate_lookup(test_name, alt))
+        cloned_alternatives_array -= [alt]
+      end
+      experiment.status = "Live"
+      experiment.save(:validate => false)
+      BananaSplit.cache.write("BananaSplit::Experiment::exists(#{test_name})".gsub(" ", "_"), 1)
+
+      #This might have issues in very, very high concurrency environments...
+
+      tests_listening_to_conversion = BananaSplit.cache.read("BananaSplit::tests_listening_to_conversion#{conversion_name}") || []
+      tests_listening_to_conversion += [test_name] unless tests_listening_to_conversion.include? test_name
+      BananaSplit.cache.write("BananaSplit::tests_listening_to_conversion#{conversion_name}", tests_listening_to_conversion)
+      experiment
+    end
+  end
+
+  def end_experiment!(final_alternative, conversion_name = nil)
+    conversion_name ||= test_name
+    ActiveRecord::Base.transaction do
+      alternatives.each do |alternative|
+        alternative.lookup = "Experiment completed.  #{alternative.id}"
+        alternative.save!
+      end
+      update_attribute(:status, "Finished")
+      BananaSplit.cache.write("BananaSplit::Experiment::short_circuit(#{test_name})".gsub(" ", "_"), final_alternative)
+    end
+  end
+
+end

data/lib/bananasplit/rails/controller/dashboard.rb

@@ -0,0 +1,13 @@
+class BananaSplit
+  module Rails
+    module Controller
+      module Dashboard
+
+        def index
+          @experiments = BananaSplit::Experiment.all
+        end
+
+      end
+    end
+  end
+end

data/lib/bananasplit/statistics.rb

@@ -0,0 +1,90 @@
+#Designed to be included into BananaSplit::Experiment, but you can feel free to adapt this
+#to anything you want.
+
+module BananaSplit::Statistics
+
+  HANDY_Z_SCORE_CHEATSHEET = [[0.10, 1.29], [0.05, 1.65], [0.01, 2.33], [0.001, 3.08]]
+
+  PERCENTAGES = {0.10 => '90%', 0.05 => '95%', 0.01 => '99%', 0.001 => '99.9%'}
+
+  DESCRIPTION_IN_WORDS = {0.10 => 'fairly confident', 0.05 => 'confident',
+                         0.01 => 'very confident', 0.001 => 'extremely confident'}
+  def zscore
+    if alternatives.size != 2
+      raise "Sorry, can't currently automatically calculate statistics for A/B tests with > 2 alternatives."
+    end
+
+    if (alternatives[0].participants == 0) || (alternatives[1].participants == 0)
+      raise "Can't calculate the z score if either of the alternatives lacks participants."
+    end
+
+    cr1 = alternatives[0].conversion_rate
+    cr2 = alternatives[1].conversion_rate
+
+    n1 = alternatives[0].participants
+    n2 = alternatives[1].participants
+
+    numerator = cr1 - cr2
+    frac1 = cr1 * (1 - cr1) / n1
+    frac2 = cr2 * (1 - cr2) / n2
+
+    numerator / ((frac1 + frac2) ** 0.5)
+  end
+
+  def p_value
+    index = 0
+    z = zscore
+    z = z.abs
+    found_p = nil
+    while index < HANDY_Z_SCORE_CHEATSHEET.size do
+      if (z > HANDY_Z_SCORE_CHEATSHEET[index][1])
+        found_p = HANDY_Z_SCORE_CHEATSHEET[index][0]
+      end
+      index += 1
+    end
+    found_p
+  end
+
+  def is_statistically_significant?(p = 0.05)
+    p_value <= p
+  end
+
+  def pretty_conversion_rate
+    sprintf("%4.2f%%", conversion_rate * 100)
+  end
+
+  def describe_result_in_words
+    begin
+      z = zscore
+    rescue
+      return "Could not execute the significance test because one or more of the alternatives has not been seen yet."
+    end
+    p = p_value
+
+    words = ""
+    if (alternatives[0].participants < 10) || (alternatives[1].participants < 10)
+      words += "Take these results with a grain of salt since your samples are so small: "
+    end
+
+    alts = alternatives - [best_alternative]
+    worst_alternative = alts.first
+
+    words += "The best alternative you have is: [#{best_alternative.content}], which had "
+    words += "#{best_alternative.conversions} conversions from #{best_alternative.participants} participants "
+    words += "(#{best_alternative.pretty_conversion_rate}).  The other alternative was [#{worst_alternative.content}], "
+    words += "which had #{worst_alternative.conversions} conversions from #{worst_alternative.participants} participants "
+    words += "(#{worst_alternative.pretty_conversion_rate}).  "
+
+    if (p.nil?)
+      words += "However, this difference is not statistically significant."
+    else
+      words += "This difference is #{PERCENTAGES[p]} likely to be statistically significant, which means you can be "
+      words += "#{DESCRIPTION_IN_WORDS[p]} that it is the result of your alternatives actually mattering, rather than "
+      words += "being due to random chance.  However, this statistical test can't measure how likely the currently "
+      words += "observed magnitude of the difference is to be accurate or not.  It only says \"better\", not \"better "
+      words += "by so much\"."
+    end
+    words
+  end
+
+end

data/lib/bananasplit/version.rb

@@ -0,0 +1,3 @@
+class BananaSplit
+  VERSION = "2.0.3"
+end

data/lib/bananasplit/views/dashboard/_experiment.erb

@@ -0,0 +1,43 @@
+<h3><%= experiment.test_name.titleize %> <%= %Q|<i>(Test completed)</i>| if experiment.status != "Live" %> </h3>
+<% short_circuit = BananaSplit.cache.read("BananaSplit::Experiment::short_circuit(#{experiment.test_name})".gsub(" ", "")) %>
+<table class="experiment" style="border: 1px black;">
+  <tr class="header_row">
+    <th>Name</th>
+    <th>Participants</th>
+    <th>Conversions</th>
+    <th>Notes</th>
+  </tr>
+  <tr class="experiment_row">
+    <td>Experiment Total: </td>
+    <td><%= experiment.participants %> </td>
+    <td><%= experiment.conversions %> (<%= experiment.pretty_conversion_rate %>)</td>
+    <td></td>
+  </tr>
+  <% experiment.alternatives.each do |alternative| %>
+  <tr class="alternative_row">
+    <td>
+        <%= h alternative.content %>
+    </td>
+    <td><%= alternative.participants %></td>
+    <td><%= alternative.conversions %> (<%= alternative.pretty_conversion_rate %>)</td>
+    <td>
+      <% unless experiment.status != "Live" %>
+      <%= link_to("End experiment, picking this.", url_for(:id => alternative.id,
+      :action => "end_experiment"),
+      :method => :post,
+      :confirm => "Are you sure you want to terminate this experiment?  This is not reversible."
+      ) %>
+      <% else %>
+        <% if alternative.content == short_circuit %>
+          <b>(All users seeing this.)</b>
+        <% end %>
+      <% end %>
+    </td>
+  </tr>
+  <% end %>
+  <tr>
+    <td colspan="4">
+      <b>Significance test results: </b><%= experiment.describe_result_in_words %>
+    </td>
+  </tr>
+</table>

data/lib/bananasplit/views/dashboard/index.erb

@@ -0,0 +1,20 @@
+<div id="abingo_dashboard">
+<p><h1>Welcome to your A/Bingo dashboard!</h1>
+
+See <a href="http://www.bingocardcreator.com/abingo">the official site</a> for documentation.
+I encourage you to customize this page to fit your needs.  See /vendor/plugins/abingo/views for
+the view templates.  Hack them to pieces -- please!
+</p>
+
+<p>
+  <% if flash[:notice] %>
+  <span style="color: green;"><%= flash[:notice] %> </span>
+  <% end %>
+  <h2>All Experiments</h2>
+  
+<% @experiments.each do |experiment| %>
+  <%= render :partial => "dashboard/experiment", :locals => {:experiment => experiment} %>
+  <br/>
+<% end %>
+</p>
+</div>

data/lib/bananasplit_sugar.rb

@@ -0,0 +1,48 @@
+#This module exists entirely to save finger strain for programmers.
+#It is designed to be included in your ApplicationController.
+#
+#See abingo.rb for descriptions of what these do.
+
+module BananaSplitSugar
+
+  def ab_test(abingo, test_name, alternatives = nil, options = {})
+    if (BananaSplit.options[:enable_specification] && !params[test_name].nil?)
+      choice = params[test_name]
+    elsif (BananaSplit.options[:enable_override_in_session] && !session[test_name].nil?)
+      choice = session[test_name]
+    elsif (alternatives.nil?)
+      choice = abingo.flip(test_name)
+    else
+      choice = abingo.test(test_name, alternatives, options)
+    end
+
+    if block_given?
+      yield(choice)
+    else
+      choice
+    end
+  end
+
+  def bingo!(abingo, test_name, options = {})
+    abingo.bingo!(test_name, options)
+  end
+
+  #Mark the user as a human.
+  def abingo_mark_human(abingo)
+    textual_result = "1"
+    begin
+      a = params[:a].to_i
+      b = params[:b].to_i
+      c = params[:c].to_i
+      if (request.method == :post && (a + b == c))
+        abingo.human!
+      else
+        textual_result = "0"
+      end
+    rescue #If a bot doesn't pass a, b, or c, to_i will fail.  This scarfs up the exception, to save it from polluting our logs.
+      textual_result = "0"
+    end
+    render :text => textual_result, :layout => false #Not actually used by browser
+  end
+
+end

data/lib/bananasplit_view_helper.rb

@@ -0,0 +1,40 @@
+#Gives you easy syntax to use ABingo in your views.
+
+module BananaSplitViewHelper
+
+  def ab_test(abingo, test_name, alternatives = nil, options = {}, &block)
+
+    if (BananaSplit.options[:enable_specification] && !params[test_name].nil?)
+      choice = params[test_name]
+    elsif (BananaSplit.options[:enable_override_in_session] && !session[test_name].nil?)
+      choice = session[test_name]
+    elsif (alternatives.nil?)
+      choice = abingo.flip(test_name)
+    else
+      choice = abingo.test(test_name, alternatives, options)
+    end
+
+    if block
+      content_tag = capture(choice, &block)
+    else
+      choice
+    end
+  end
+
+  def bingo!(abingo, test_name, options = {})
+    abingo.bingo!(test_name, options)
+  end
+
+  #This causes an AJAX post against the URL.  That URL should call BananaSplit.human!
+  #This guarantees that anyone calling BananaSplit.human! is capable of at least minimal Javascript execution, and thus is (probably) not a robot.
+  def include_humanizing_javascript(url = "/abingo_mark_human", style = :prototype)
+    script = nil
+    if (style == :prototype)
+      script = "var a=Math.floor(Math.random()*11); var b=Math.floor(Math.random()*11);var x=new Ajax.Request('#{url}', {parameters:{a: a, b: b, c: a+b}})"
+    elsif (style == :jquery)
+      script = "var a=Math.floor(Math.random()*11); var b=Math.floor(Math.random()*11);var x=jQuery.post('#{url}', {a: a, b: b, c: a+b})"
+    end
+    script.nil? ? "" : %Q|<script type="text/javascript">#{script}</script>|
+  end
+
+end

data/test/abingo_test.rb

@@ -0,0 +1,220 @@
+require_relative 'test_helper'
+
+class BananaSplitTest < Test::Unit::TestCase
+
+  setup do
+    BananaSplit.options = {}
+  end
+
+  teardown do
+    BananaSplit.cache.clear
+    BananaSplit::Experiment.delete_all
+    BananaSplit::Alternative.delete_all
+  end
+
+  test "identity automatically assigned" do
+    split = BananaSplit.identify
+    assert split.identity != nil
+  end
+
+  test "alternative parsing" do
+    array = %w{a b c}
+    assert_equal array, BananaSplit.parse_alternatives(array)
+    assert_equal 65, BananaSplit.parse_alternatives(65).size
+    assert_equal 4, BananaSplit.parse_alternatives(2..5).size
+    assert !(BananaSplit.parse_alternatives(2..5).include? 1)
+  end
+
+  test "experiment creation" do
+    assert_equal 0, BananaSplit::Experiment.count
+    assert_equal 0, BananaSplit::Alternative.count
+    alternatives = %w{A B}
+    split = BananaSplit.identify
+    alternative_selected = split.test("unit_test_sample_A", alternatives)
+    assert_equal 1, BananaSplit::Experiment.count
+    assert_equal 2, BananaSplit::Alternative.count
+    assert alternatives.include?(alternative_selected)
+  end
+
+  test "exists works right" do
+    split = BananaSplit.identify
+    split.test("exist works right", %w{does does_not})
+    assert BananaSplit::Experiment.exists?("exist works right")
+  end
+
+  test "alternatives picked consistently" do
+    split = BananaSplit.identify
+    alternative_picked = split.test("consistency_test", 1..100)
+    100.times do
+      assert_equal alternative_picked, split.test("consistency_test", 1..100)
+    end
+  end
+
+  test "participation works" do
+    new_tests = %w{participationA participationB participationC}
+    split = BananaSplit.identify
+    new_tests.map do |test_name|
+      split.test(test_name, 1..5)
+    end
+
+    participating_tests = BananaSplit.cache.read("BananaSplit::participating_tests::#{split.identity}") || []
+
+    new_tests.map do |test_name|
+      assert participating_tests.include? test_name
+    end
+  end
+
+  test "participants counted" do
+    test_name = "participants_counted_test"
+    split = BananaSplit.identify
+    alternative = split.test(test_name, %w{a b c})
+
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    lookup = BananaSplit::Alternative.calculate_lookup(test_name, alternative)
+    chosen_alt = BananaSplit::Alternative.find_by_lookup(lookup)
+    assert_equal 1, ex.participants
+    assert_equal 1, chosen_alt.participants
+  end
+
+  test "conversion tracking by test name" do
+    test_name = "conversion_test_by_name"
+    split = BananaSplit.identify
+    alternative = split.test(test_name, %w{a b c})
+    split.bingo!(test_name)
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    lookup = BananaSplit::Alternative.calculate_lookup(test_name, alternative)
+    chosen_alt = BananaSplit::Alternative.find_by_lookup(lookup)
+    assert_equal 1, ex.conversions
+    assert_equal 1, chosen_alt.conversions
+    split.bingo!(test_name)
+
+    #Should still only have one because this conversion should not be double counted.
+    #We haven't specified that in the test options.
+    assert_equal 1, BananaSplit::Experiment.find_by_test_name(test_name).conversions
+  end
+
+  test "conversion tracking by conversion name" do
+    split = BananaSplit.identify
+    conversion_name = "purchase"
+    tests = %w{conversionTrackingByConversionNameA conversionTrackingByConversionNameB conversionTrackingByConversionNameC}
+    tests.map do |test_name|
+      split.test(test_name, %w{A B}, :conversion => conversion_name)
+    end
+
+    split.bingo!(conversion_name)
+    tests.map do |test_name|
+      assert_equal 1, BananaSplit::Experiment.find_by_test_name(test_name).conversions
+    end
+  end
+
+  test "short circuiting works" do
+    conversion_name = "purchase"
+    test_name = "short circuit test"
+    split = BananaSplit.identify
+    alt_picked = split.test(test_name, %w{A B}, :conversion => conversion_name)
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    alt_not_picked = (%w{A B} - [alt_picked]).first
+
+    ex.end_experiment!(alt_not_picked, conversion_name)
+
+    ex.reload
+    assert_equal "Finished", ex.status
+
+    split.bingo!(test_name)  #Should not be counted, test is over.
+    assert_equal 0, ex.conversions
+
+    new_bingo = BananaSplit.identify("shortCircuitTestNewIdentity")
+    new_bingo.test(test_name, %w{A B}, :conversion => conversion_name)
+    ex.reload
+    assert_equal 1, ex.participants  #Original identity counted, new identity not counted b/c test stopped
+  end
+
+  test "proper experiment creation in high concurrency" do
+    conversion_name = "purchase"
+    test_name = "high_concurrency_test"
+    alternatives = %w{foo bar}
+
+    threads = []
+    5.times do
+      threads << Thread.new do
+        split = BananaSplit.identify
+        split.test(test_name, alternatives, :conversion => conversion_name)
+        ActiveRecord::Base.connection.close
+      end
+    end
+    threads.each(&:join)
+    assert_equal 1, BananaSplit::Experiment.count_by_sql(["select count(id) from experiments where test_name = ?", test_name])
+  end
+
+  test "proper conversions with concurrency" do
+    test_name = "conversion_concurrency_test"
+    alternatives = %w{foo bar}
+    threads = []
+    alternatives.size.times do |i|
+      threads << Thread.new do
+        split = BananaSplit.identify(i)
+        split.test(test_name, alternatives)
+        split.bingo!(test_name)
+        sleep(0.3) if i == 0
+        ActiveRecord::Base.connection.close
+      end
+    end
+    threads.each(&:join)
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    ex.alternatives.each do |alternative|
+      assert_equal 1, alternative.conversions
+    end
+  end
+
+  test "non-humans are ignored for participation and conversions if not explicitly counted" do
+    BananaSplit.options[:count_humans_only] = true
+    BananaSplit.options[:expires_in] = 1.hour
+    BananaSplit.options[:expires_in_for_bots] = 3.seconds
+    first_identity = BananaSplit.identify("unsure_if_human#{Time.now.to_i}")
+    test_name = "are_you_a_human"
+    first_identity.test(test_name, %w{does_not matter})
+
+    assert !first_identity.is_human?, "Identity not marked as human yet."
+
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    first_identity.bingo!(test_name)
+    assert_equal 0, ex.participants, "Not human yet, so should have no participants."
+    assert_equal 0, ex.conversions, "Not human yet, so should have no conversions."
+
+    first_identity.human!
+
+    #Setting up second participant who doesn't convert.
+    second_identity = BananaSplit.identify("unsure_if_human_2_#{Time.now.to_i}")
+    second_identity.test(test_name, %w{does_not matter})
+    second_identity.human!
+
+    ex = BananaSplit::Experiment.find_by_test_name(test_name)
+    assert_equal 2, ex.participants, "Now that we're human, our participation should matter."
+    assert_equal 1, ex.conversions, "Now that we're human, our conversions should matter, but only one of us converted."
+  end
+
+  test "Participating tests for a given identity" do
+    split = BananaSplit.identify("test_participant")
+    test_names = (1..3).map {|t| "participating_test_test_name #{t}"}
+    test_alternatives = %w{yes no}
+    test_names.each {|test_name| split.test(test_name, test_alternatives)}
+    ex = BananaSplit::Experiment.last
+    ex.end_experiment!("no")  #End final of 3 tests, leaving 2 presently running
+
+    assert_equal 2, split.participating_tests.size  #Pairs for two tests
+    split.participating_tests.each do |key, value|
+      assert test_names.include? key
+      assert test_alternatives.include? value
+    end
+
+    assert_equal 3, split.participating_tests(false).size #pairs for three tests
+    split.participating_tests(false).each do |key, value|
+      assert test_names.include? key
+      assert test_alternatives.include? value
+    end
+
+    non_participant = BananaSplit.identify("test_nonparticipant")
+    assert_equal({}, non_participant.participating_tests)
+  end
+
+end