mikeg-vanity 1.3.0

data/test/experiment_test.rb

@@ -0,0 +1,136 @@
+require "test/test_helper"
+
+class ExperimentTest < Test::Unit::TestCase
+
+  def setup
+    super
+    metric "Happiness"
+  end
+
+  # -- Defining experiment --
+  
+  def test_can_access_experiment_by_id
+    exp = new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_equal exp, experiment(:ice_cream_flavor)
+  end
+
+  def test_fail_when_defining_same_experiment_twice
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+        end
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+        end
+      RUBY
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_uses_playground_namespace_for_experiment
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_equal "vanity:#{Vanity::Version::MAJOR}:ice_cream_flavor", experiment(:ice_cream_flavor).send(:key)
+    assert_equal "vanity:#{Vanity::Version::MAJOR}:ice_cream_flavor:participants", experiment(:ice_cream_flavor).send(:key, "participants")
+  end
+
+
+  # -- Loading experiments --
+
+  def test_fails_if_cannot_load_named_experiment
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_loading_experiment
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          def xmts
+            "x"
+          end
+          metrics :happiness
+        end
+      RUBY
+    end
+    assert_equal "x", experiment(:ice_cream_flavor).xmts
+  end
+
+  def test_fails_if_error_loading_experiment
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write "fail 'yawn!'"
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_complains_if_not_defined_where_expected
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write ""
+    end
+    assert_raises NameError do
+      experiment(:ice_cream_flavor)
+    end
+  end
+
+  def test_reloading_experiments
+    new_ab_test(:ab) { metrics :happiness }
+    new_ab_test(:cd) { metrics :happiness }
+    assert 2, Vanity.playground.experiments.size
+    Vanity.playground.reload!
+    assert Vanity.playground.experiments.empty?
+  end
+
+
+  # -- Attributes --
+
+  def test_experiment_mapping_name_to_id
+    experiment = new_ab_test("Ice Cream Flavor/Tastes") { metrics :happiness }
+    assert_equal "Ice Cream Flavor/Tastes", experiment.name
+    assert_equal :ice_cream_flavor_tastes, experiment.id
+  end
+
+  def test_saving_experiment_after_definition
+    File.open "tmp/experiments/ice_cream_flavor.rb", "w" do |f|
+      f.write <<-RUBY
+        ab_test "Ice Cream Flavor" do
+          metrics :happiness
+          expects(:save)
+        end
+      RUBY
+    end
+    Vanity.playground.experiment(:ice_cream_flavor)
+  end
+
+  def test_experiment_has_created_timestamp
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_instance_of Time, experiment(:ice_cream_flavor).created_at
+    assert_in_delta experiment(:ice_cream_flavor).created_at.to_i, Time.now.to_i, 1
+  end
+ 
+  def test_experiment_keeps_created_timestamp_across_definitions
+    past = Date.today - 1
+    Timecop.travel past do
+      new_ab_test(:ice_cream_flavor) { metrics :happiness }
+      assert_equal past.to_time.to_i, experiment(:ice_cream_flavor).created_at.to_i
+    end
+
+    new_playground
+    metric :happiness
+    new_ab_test(:ice_cream_flavor) { metrics :happiness }
+    assert_equal past.to_time.to_i, experiment(:ice_cream_flavor).created_at.to_i
+  end
+
+  def test_experiment_has_description
+    new_ab_test :ice_cream_flavor do
+      description "Because 31 is not enough ..."
+      metrics :happiness
+    end
+    assert_equal "Because 31 is not enough ...", experiment(:ice_cream_flavor).description
+  end
+
+end

data/test/experiments/age_and_zipcode.rb

@@ -0,0 +1,19 @@
+ab_test "Age and Zipcode" do
+  description <<-TEXT
+Testing new registration form that asks for age and zipcode. Option A presents
+the existing form, and option B adds age and zipcode fields.
+   
+We know option B will convert less, but higher quality leads. If we lose less
+than 20% conversions, we're going to switch to option B.
+  TEXT
+  metrics :signups
+
+  complete_if do
+    alternatives.all? { |alt| alt.participants > 100 }
+  end
+  outcome_is do
+    one_field = alternative(false)
+    three_fields = alternative(true)
+    three_fields.conversion_rate >= 0.8 * one_field.conversion_rate ? three_fields : one_field
+  end
+end

data/test/experiments/metrics/cheers.rb

@@ -0,0 +1,3 @@
+metric "Cheers" do
+  description "They love us, don't they?"
+end

data/test/experiments/metrics/signups.rb

@@ -0,0 +1,2 @@
+metric "Signups" do
+end

data/test/experiments/metrics/yawns.rb

@@ -0,0 +1,3 @@
+metric "Yawns" do
+  description "How many yawns/sec can our video-sharing micro-blogging social network elicit?"
+end

data/test/experiments/null_abc.rb

@@ -0,0 +1,5 @@
+ab_test "Null/ABC" do
+  description "Testing three alternatives"
+  alternatives nil, :red, :green, :blue
+  metrics :signups
+end

data/test/metric_test.rb

@@ -0,0 +1,518 @@
+require "test/test_helper"
+
+class Sky < ActiveRecord::Base
+  connection.drop_table :skies if table_exists?
+  connection.create_table :skies do |t|
+    t.integer :height
+    t.timestamps
+  end
+
+  named_scope :high, lambda { { :conditions=>"height >= 4" } }
+end
+
+
+context "Metric" do
+
+  # -- Via the playground --
+
+  context "playground" do
+
+    test "knows all loaded metrics" do
+      metric "Yawns/sec", "Cheers/sec"
+      assert Vanity.playground.metrics.keys.include?(:yawns_sec)
+      assert Vanity.playground.metrics.keys.include?(:cheers_sec)
+    end
+
+    test "loads metric definitions" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Yawns/sec" do
+            def xmts
+              "x"
+            end
+          end
+        RUBY
+      end
+      assert_equal "x", Vanity.playground.metric(:yawns_sec).xmts
+    end
+
+    test "bubbles up loaded metrics" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write "fail 'yawn!'"
+      end
+      assert_raises NameError do
+        Vanity.playground.metric(:yawns_sec)
+      end
+    end
+
+    test "map identifier from file name" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "yawns/hour" do
+          end
+        RUBY
+      end
+      assert Vanity.playground.metric(:yawns_sec)
+    end
+
+    test "fails tracking unknown metric" do
+      assert_raises NameError do
+        Vanity.playground.track! :yawns_sec
+      end
+    end
+
+    test "reloading metrics" do
+      metric "Yawns/sec", "Cheers/sec"
+      Vanity.playground.metric(:yawns_sec)
+      Vanity.playground.metric(:cheers_sec)
+      assert 2, Vanity.playground.metrics.size
+      metrics = Vanity.playground.metrics.values
+      Vanity.playground.reload!
+      assert 2, Vanity.playground.metrics.size
+      assert_not_equal metrics, Vanity.playground.metrics.values
+    end
+
+    test "ignores undefined metrics in database" do
+      metric "Yawns/sec"
+      Vanity.playground.reload!
+      assert Vanity.playground.metrics.empty?
+    end
+
+  end
+
+
+  # -- Tracking --
+
+  context "tracking" do
+    test "can count" do
+      metric "Yawns/sec", "Cheers/sec"
+      4.times { Vanity.playground.track! :yawns_sec }
+      2.times { Vanity.playground.track! :cheers_sec }
+      yawns = Vanity.playground.metric(:yawns_sec).values(today, today).first
+      cheers = Vanity.playground.metric(:cheers_sec).values(today, today).first
+      assert yawns = 2 * cheers
+    end
+
+    test "can tell the time" do
+      metric "Yawns/sec"
+      Timecop.travel(today - 4) { 4.times { Vanity.playground.track! :yawns_sec } }
+      Timecop.travel(today - 2) { 2.times { Vanity.playground.track! :yawns_sec } }
+      1.times { Vanity.playground.track! :yawns_sec }
+      boredom = Vanity.playground.metric(:yawns_sec).values(today - 5, today)
+      assert_equal [0,4,0,2,0,1], boredom
+    end
+
+    test "with no value" do
+      metric "Yawns/sec", "Cheers/sec", "Looks"
+      Vanity.playground.track! :yawns_sec, 0
+      Vanity.playground.track! :cheers_sec, -1
+      Vanity.playground.track! :looks, 10
+      assert_equal 0, Vanity.playground.metric(:yawns_sec).values(today, today).sum
+      assert_equal 0, Vanity.playground.metric(:cheers_sec).values(today, today).sum
+      assert_equal 10, Vanity.playground.metric(:looks).values(today, today).sum
+    end
+
+    test "with count" do
+      metric "Yawns/sec"
+      Timecop.travel(today - 4) { Vanity.playground.track! :yawns_sec, 4 }
+      Timecop.travel(today - 2) { Vanity.playground.track! :yawns_sec, 2 }
+      Vanity.playground.track! :yawns_sec
+      boredom = Vanity.playground.metric(:yawns_sec).values(today - 5, today)
+      assert_equal [0,4,0,2,0,1], boredom
+    end
+
+    test "runs hook" do
+      metric "Many Happy Returns"
+      total = 0
+      Vanity.playground.metric(:many_happy_returns).hook do |metric_id, timestamp, count|
+        assert_equal :many_happy_returns, metric_id
+        assert_in_delta Time.now.to_i, timestamp.to_i, 1
+        total += count
+      end
+      Vanity.playground.track! :many_happy_returns, 6
+      assert_equal 6, total
+    end
+
+    test "runs multiple hooks" do
+      metric "Many Happy Returns"
+      returns = 0
+      Vanity.playground.metric(:many_happy_returns).hook { returns += 1 }
+      Vanity.playground.metric(:many_happy_returns).hook { returns += 1 }
+      Vanity.playground.metric(:many_happy_returns).hook { returns += 1 }
+      Vanity.playground.track! :many_happy_returns
+      assert_equal 3, returns
+    end
+
+    test "destroy wipes metrics" do
+      metric "Many Happy Returns"
+      Vanity.playground.track! :many_happy_returns, 3
+      assert_equal [3], Vanity.playground.metric(:many_happy_returns).values(today, today)
+      Vanity.playground.metric(:many_happy_returns).destroy!
+      assert_equal [0], Vanity.playground.metric(:many_happy_returns).values(today, today)
+    end
+  end
+
+
+  # -- Metric name --
+
+  context "name" do
+    test "can be whatever" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Yawns per second" do
+          end
+        RUBY
+      end
+      assert_equal "Yawns per second", Vanity.playground.metric(:yawns_sec).name
+    end
+  end  
+
+
+  # -- Description helper --
+
+  context "description" do
+    test "metric with description" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Yawns/sec" do
+            description "Am I that boring?"
+          end
+        RUBY
+      end
+      assert_equal "Am I that boring?", Vanity::Metric.description(Vanity.playground.metric(:yawns_sec))
+    end
+
+    test "metric without description" do
+      File.open "tmp/experiments/metrics/yawns_sec.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Yawns/sec" do
+          end
+        RUBY
+      end
+      assert_nil Vanity::Metric.description(Vanity.playground.metric(:yawns_sec))
+    end
+
+    test "metric with no method description" do
+      metric = Object.new
+      assert_nil Vanity::Metric.description(metric)
+    end
+  end
+
+
+  # -- Metric bounds --
+
+  context "bounds" do
+    test "metric with bounds" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            def bounds
+              [6,12]
+            end
+          end
+        RUBY
+      end
+      assert_equal [6,12], Vanity::Metric.bounds(Vanity.playground.metric(:sky_is_limit))
+    end
+
+    test "metric without bounds" do
+      metric "Sky is limit"
+      assert_equal [nil, nil], Vanity::Metric.bounds(Vanity.playground.metric(:sky_is_limit))
+    end
+
+    test "metric with no method bounds" do
+      metric = Object.new
+      assert_equal [nil, nil], Vanity::Metric.bounds(metric)
+    end
+  end
+
+
+  # -- Timestamp --
+ 
+  context "created_at" do
+    test "for new metric" do
+      metric "Coolness"
+      metric = Vanity.playground.metric(:coolness)
+      assert_instance_of Time, metric.created_at
+      assert_in_delta metric.created_at.to_i, Time.now.to_i, 1
+    end
+
+    test "across restarts" do
+      past = Date.today - 1
+      Timecop.travel past do
+        metric "Coolness"
+        coolness = Vanity.playground.metric(:coolness)
+        assert_in_delta coolness.created_at.to_i, past.to_time.to_i, 1
+      end
+
+      new_playground
+      metric "Coolness"
+      new_cool = Vanity.playground.metric(:coolness)
+      assert_in_delta new_cool.created_at.to_i, past.to_time.to_i, 1
+    end
+  end 
+
+
+  # -- Data --
+
+  context "data" do
+    test "explicit dates" do
+      metric "Yawns/sec"
+      Timecop.travel(today - 4) { Vanity.playground.track! :yawns_sec, 4 }
+      Timecop.travel(today - 2) { Vanity.playground.track! :yawns_sec, 2 }
+      Vanity.playground.track! :yawns_sec
+      boredom = Vanity::Metric.data(Vanity.playground.metric(:yawns_sec), Date.today - 5, Date.today)
+      assert_equal [[today - 5, 0], [today - 4, 4], [today - 3, 0], [today - 2, 2], [today - 1, 0], [today, 1]], boredom
+    end
+
+    test "start date only" do
+      metric "Yawns/sec"
+      Timecop.travel(today - 4) { Vanity.playground.track! :yawns_sec, 4 }
+      Timecop.travel(today - 2) { Vanity.playground.track! :yawns_sec, 2 }
+      Vanity.playground.track! :yawns_sec
+      boredom = Vanity::Metric.data(Vanity.playground.metric(:yawns_sec), Date.today - 4)
+      assert_equal [[today - 4, 4], [today - 3, 0], [today - 2, 2], [today - 1, 0], [today, 1]], boredom
+    end
+
+    test "start date and duration" do
+      metric "Yawns/sec"
+      Timecop.travel(today - 4) { Vanity.playground.track! :yawns_sec, 4 }
+      Timecop.travel(today - 2) { Vanity.playground.track! :yawns_sec, 2 }
+      Vanity.playground.track! :yawns_sec
+      boredom = Vanity::Metric.data(Vanity.playground.metric(:yawns_sec), 5)
+      assert_equal [[today - 4, 4], [today - 3, 0], [today - 2, 2], [today - 1, 0], [today, 1]], boredom
+    end
+
+    test "no data" do
+      metric "Yawns/sec"
+      boredom = Vanity::Metric.data(Vanity.playground.metric(:yawns_sec))
+      assert_equal 90, boredom.size
+      assert_equal [today - 89, 0], boredom.first
+      assert_equal [today, 0], boredom.last
+    end
+
+    test "using custom values method" do
+      File.open "tmp/experiments/metrics/hours_in_day.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Hours in day" do
+            def values(from, to)
+              (from..to).map { |d| 24 }
+            end
+          end
+        RUBY
+      end
+      data = Vanity::Metric.data(Vanity.playground.metric(:hours_in_day))
+      assert_equal [24] * 90, data.map(&:last)
+    end
+
+  end
+
+
+  # -- ActiveRecord --
+
+  context "ActiveRecord" do
+
+    test "record count" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      Sky.create!
+      assert_equal 1, Sky.count
+      assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+    test "record sum" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :sum=>:height
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      Sky.create! :height=>4
+      Sky.create! :height=>2
+      assert_equal 6, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+    test "record average" do
+      Sky.aggregates
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :average=>:height
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      Sky.create! :height=>4
+      Sky.create! :height=>2
+      assert_equal 3, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+    test "record minimum" do
+      Sky.aggregates
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :minimum=>:height
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      Sky.create! :height=>4
+      Sky.create! :height=>2
+      assert_equal 2, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+    test "record maximum" do
+      Sky.aggregates
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :maximum=>:height
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      Sky.create! :height=>4
+      Sky.create! :height=>2
+      assert_equal 4, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+    test "with conditions" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :sum=>:height, :conditions=>["height > 4"]
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      high_skies = 0
+      metric(:sky_is_limit).hook do |metric_id, timestamp, height|
+        assert height > 4
+        high_skies += height
+      end
+      [nil,5,3,6].each do |height|
+        Sky.create! :height=>height
+      end
+      assert_equal 11, Vanity::Metric.data(metric(:sky_is_limit)).sum(&:last)
+      assert_equal 11, high_skies
+    end
+
+    test "with scope" do
+      Sky.aggregates
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky.high
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      total = 0
+      metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+        total += count
+      end
+      Sky.create! :height=>4
+      Sky.create! :height=>2
+      assert_equal 1, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+      assert_equal 1, total
+    end
+
+    test "hooks" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :sum=>:height
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      total = 0
+      metric(:sky_is_limit).hook do |metric_id, timestamp, count|
+        assert_equal :sky_is_limit, metric_id
+        assert_in_delta Time.now.to_i, timestamp.to_i, 1
+        total += count
+      end
+      Sky.create! :height=>4
+      assert_equal 4, total
+    end
+
+    test "after_create not after_save" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      once = nil
+      metric(:sky_is_limit).hook do
+        fail "Metric tracked twice" if once
+        once = true
+      end
+      Sky.create!
+      Sky.last.update_attributes :height=>4
+    end
+
+    test "with after_save" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            model Sky, :conditions=>["height > 3"]
+            Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      times = 0
+      metric(:sky_is_limit).hook do
+        times += 1
+      end
+      Sky.create!
+      (1..5).each do |height|
+        Sky.last.update_attributes! :height=>height
+      end
+      assert_equal 2, times
+    end
+
+    test "do it youself" do
+      File.open "tmp/experiments/metrics/sky_is_limit.rb", "w" do |f|
+        f.write <<-RUBY
+          metric "Sky is limit" do
+            Sky.after_save { |sky| track! if sky.height_changed? && sky.height > 3 }
+          end
+        RUBY
+      end
+      Vanity.playground.metrics
+      (1..5).each do |height|
+        Sky.create! :height=>height
+      end
+      Sky.first.update_attributes! :height=>4
+      assert_equal 3, Vanity::Metric.data(metric(:sky_is_limit)).last.last
+    end
+
+  end
+
+  
+  # -- Helper methods --
+
+  def today
+    @today ||= Date.today
+  end
+
+  teardown do
+    Sky.delete_all
+    Sky.after_create.clear
+    Sky.after_save.clear
+  end
+
+end