moses-vanity 1.7.1

data/test/experiment/base_test.rb

@@ -0,0 +1,150 @@
+require "test/test_helper"
+
+class ExperimentTest < Test::Unit::TestCase
+
+  def setup
+    super
+    metric "Happiness"
+  end
+
+  # -- Defining experiment --
+  
+  def test_can_access_experiment_by_id
+    exp = new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_equal exp, experiment(:ice_cream_flavor)
+  end
+
+  def test_fail_when_defining_same_experiment_twice
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+        end
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+        end
+      RUBY
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+
+  # -- Loading experiments --
+
+  def test_fails_if_cannot_load_named_experiment
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_loading_experiment
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          def xmts
+            "x"
+          end
+          metrics :happiness
+        end
+      RUBY
+    end
+    assert_equal "x", experiment(:ice_cream_flavor).xmts
+  end
+
+  def test_fails_if_error_loading_experiment
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write "fail 'yawn!'"
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_complains_if_not_defined_where_expected
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write ""
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_reloading_experiments
+    new_ab_test(:ab) { metrics :happiness }
+    new_ab_test(:cd) { metrics :happiness }
+    assert_equal 2, Vanity.playground.experiments.size
+    Vanity.playground.reload!
+    assert Vanity.playground.experiments.empty?
+  end
+
+
+  # -- Attributes --
+
+  def test_experiment_mapping_name_to_id
+    experiment = new_ab_test("Ice Cream Flavor/Tastes") { metrics :happiness }
+    assert_equal "Ice Cream Flavor/Tastes", experiment.name
+    assert_equal :ice_cream_flavor_tastes, experiment.id
+  end
+
+  def test_saving_experiment_after_definition
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+          expects(:save)
+        end
+      RUBY
+    end
+    Vanity.playground.experiment(:ice_cream_flavor)
+  end
+
+  def test_experiment_has_created_timestamp
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_instance_of Time, experiment(:ice_cream_flavor).created_at
+    assert_in_delta experiment(:ice_cream_flavor).created_at.to_i, Time.now.to_i, 1
+  end
+ 
+  def test_experiment_keeps_created_timestamp_across_definitions
+    past = Date.today - 1
+    Timecop.freeze past do
+      new_ab_test(:ice_cream_flavor) { metrics :happiness }
+      assert_equal past.to_time.to_i, experiment(:ice_cream_flavor).created_at.to_i
+    end
+
+    new_playground
+    metric :happiness
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_equal past.to_time.to_i, experiment(:ice_cream_flavor).created_at.to_i
+  end
+
+  def test_experiment_has_description
+    new_ab_test :ice_cream_flavor do
+      description "Because 31 is not enough ..."
+      metrics :happiness
+    end
+    assert_equal "Because 31 is not enough ...", experiment(:ice_cream_flavor).description
+  end
+
+  def test_experiment_stores_nothing_when_collection_disabled
+    not_collecting!
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    experiment(:ice_cream_flavor).complete!
+  end
+
+  # -- completion -- #
+  
+  # check_completion is called by derived classes, but since it's
+  # part of the base_test I'm testing it here.
+  def test_error_in_check_completion
+    new_ab_test(:ab) { metrics :happiness }
+    e = experiment(:ab)
+    e.complete_if { true }
+    e.stubs(:complete!).raises(RuntimeError, "A forced error")
+    e.expects(:warn)
+    e.stubs(:identity).returns(:b)
+    e.track!(:a, Time.now, 10)
+  end
+ 
+end

data/test/experiments/age_and_zipcode.rb

@@ -0,0 +1,19 @@
+ab_test "Age and Zipcode" do
+  description <<-TEXT
+Testing new registration form that asks for age and zipcode. Option A presents
+the existing form, and option B adds age and zipcode fields.
+   
+We know option B will convert less, but higher quality leads. If we lose less
+than 20% conversions, we're going to switch to option B.
+  TEXT
+  metrics :signups
+
+  complete_if do
+    alternatives.all? { |alt| alt.participants > 100 }
+  end
+  outcome_is do
+    one_field = alternative(false)
+    three_fields = alternative(true)
+    three_fields.conversion_rate >= 0.8 * one_field.conversion_rate ? three_fields : one_field
+  end
+end

data/test/experiments/metrics/cheers.rb

@@ -0,0 +1,3 @@
+metric "Cheers" do
+  description "They love us, don't they?"
+end

data/test/experiments/metrics/signups.rb

@@ -0,0 +1,2 @@
+metric "Signups" do
+end

data/test/experiments/metrics/yawns.rb

@@ -0,0 +1,3 @@
+metric "Yawns" do
+  description "How many yawns/sec can our video-sharing micro-blogging social network elicit?"
+end

data/test/experiments/null_abc.rb

@@ -0,0 +1,5 @@
+ab_test "Null/ABC" do
+  description "Testing three alternatives"
+  alternatives nil, :red, :green, :blue
+  metrics :signups
+end

data/test/metric/active_record_test.rb

@@ -0,0 +1,277 @@
+require "test/test_helper"
+
+class Sky < ActiveRecord::Base
+  connection.drop_table :skies if table_exists?
+  connection.create_table :skies do |t|
+    t.integer :height
+    t.timestamps
+  end
+
+  named_scope :high, lambda { { :conditions=>"height >= 4" } }
+end
+
+
+context "ActiveRecord Metric" do
+
+  test "record count" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create!
+    assert_equal 1, Sky.count
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record sum" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 6, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record average" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :average=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>8
+    Sky.create! :height=>2
+    assert_equal 5, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record minimum" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :minimum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 2, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "record maximum" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :maximum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 4, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "with conditions" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height, :conditions=>["height > 4"]
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    high_skies = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, height|
+      assert height > 4
+      high_skies += height
+    end
+    [nil,5,3,6].each do |height|
+      Sky.create! :height=>height
+    end
+    assert_equal 11, Vanity::Metric.data(metric(:sky_is_limit)).sum(&:last)
+    assert_equal 11, high_skies
+  end
+
+  test "with scope" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky.high
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      total += count
+    end
+    Sky.create! :height=>4
+    Sky.create! :height=>2
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    assert_equal 1, total
+  end
+
+  test "with timestamp" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :timestamp => :created_at
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create!
+    assert_equal 1, Sky.count
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "with timestamp and table" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :timestamp => 'skies.created_at'
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    Sky.create!
+    assert_equal 1, Sky.count
+    assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "hooks" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      assert_equal :sky_is_limit, metric_id
+      assert_in_delta Time.now.to_i, timestamp.to_i, 1
+      total += count
+    end
+    Sky.create! :height=>4
+    assert_equal 4, total
+  end
+
+  test "no hooks when metrics disabled" do
+    not_collecting!
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :sum=>:height
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    total = 0
+    metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+      total += count
+    end
+    Sky.create! :height=>4
+    assert_equal 0, total
+  end
+
+  test "after_create not after_save" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    once = nil
+    metric(:sky_is_limit).hook do
+      fail "Metric tracked twice" if once
+      once = true
+    end
+    Sky.create!
+    Sky.last.update_attributes :height=>4
+  end
+
+  test "with after_save" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky, :conditions=>["height > 3"]
+          Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    times = 0
+    metric(:sky_is_limit).hook do
+      times += 1
+    end
+    Sky.create!
+    (1..5).each do |height|
+      Sky.last.update_attributes! :height=>height
+    end
+    assert_equal 2, times
+  end
+
+  test "do it youself" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+        end
+      RUBY
+    end
+    Vanity.playground.metrics
+    (1..5).each do |height|
+      Sky.create! :height=>height
+    end
+    Sky.first.update_attributes! :height=>4
+    assert_equal 3, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+  end
+
+  test "last update for new metric" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    assert_nil metric(:sky_is_limit).last_update_at
+  end
+
+  test "last update with records" do
+    File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+      f.write <<-RUBY
+        metric "Sky is limit" do
+          model Sky
+        end
+      RUBY
+    end
+    Sky.create! :height=>1
+    Timecop.freeze Time.now + 1.day do
+      Sky.create! :height=>1
+    end
+    assert_in_delta metric(:sky_is_limit).last_update_at.to_i, (Time.now + 1.day).to_i, 1
+  end
+
+  teardown do
+    Sky.delete_all
+    Sky.after_create.clear
+    Sky.after_save.clear
+  end
+end