vanity 1.8.4 → 1.9.0.beta

data/lib/vanity/templates/vanity.css

@@ -24,3 +24,5 @@
 .vanity form#milestones label { margin-right: .5em }
 .vanity form#milestones input { vertical-align: bottom }
 .vanity .metric .marking.label { position: absolute; bottom: 2em; color: #c66; font-size: 80% }
+
+.vanity .alert { padding: 8px 35px 8px 14px; margin-bottom: 20px; text-shadow: 0px 1px 0px rgba(255, 255, 255, 0.5); background-color: rgb(252, 248, 227); border: 1px solid rgb(251, 238, 213); border-radius: 4px 4px 4px 4px; }

data/lib/vanity/version.rb

@@ -1,5 +1,5 @@
 module Vanity
-  VERSION = "1.8.4"
+  VERSION = "1.9.0.beta"
 
   module Version
     version = VERSION.to_s.split(".").map { |i| i.to_i }

data/test/adapters/redis_adapter_test.rb

@@ -1,6 +1,12 @@
 require 'test/test_helper'
 
 class RedisAdapterTest < Test::Unit::TestCase
+  def setup
+    require "vanity/adapters/redis_adapter"
+    require "redis"
+    require "redis/namespace"
+  end
+
   def test_warn_on_disconnect_error
     if defined?(Redis)
       assert_nothing_raised do
@@ -8,10 +14,109 @@ class RedisAdapterTest < Test::Unit::TestCase
         mocked_redis = stub("Redis")
         mocked_redis.expects(:client).raises(RuntimeError)
         redis_adapter = Vanity::Adapters::RedisAdapter.new({})
-        redis_adapter.expects(:warn).with("Error while disconnecting from redis: RuntimeError")
         redis_adapter.stubs(:redis).returns(mocked_redis)
+        redis_adapter.expects(:warn).with("Error while disconnecting from redis: RuntimeError")
         redis_adapter.disconnect!
       end
     end
   end
+
+  def stub_redis
+    Vanity.playground.failover_on_datastore_error!
+    mocked_redis = stub("Redis")
+    redis_adapter = Vanity::Adapters::RedisAdapter.new(:redis => mocked_redis)
+
+    [redis_adapter, mocked_redis]
+  end
+
+  def test_graceful_failure_metric_track
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:incrby).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.metric_track("metric", Time.now.to_s, "3ff62e2fb51f0b22646a342a2d357aec", [1])
+    end
+  end
+
+  def test_graceful_failure_set_experiment_created_at
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:setnx).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.set_experiment_created_at("price_options", Time.now)
+    end
+  end
+
+  def test_graceful_failure_is_experiment_completed?
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:exists).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.is_experiment_completed?("price_options")
+    end
+  end
+
+  def test_graceful_failure_ab_show
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:[]=).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_show("price_options", "3ff62e2fb51f0b22646a342a2d357aec", 0)
+    end
+  end
+
+  def test_graceful_failure_ab_showing
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:[]).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_showing("price_options", "3ff62e2fb51f0b22646a342a2d357aec")
+    end
+  end
+
+  def test_graceful_failure_ab_not_showing
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:del).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_not_showing("price_options", "3ff62e2fb51f0b22646a342a2d357aec")
+    end
+  end
+
+  def test_graceful_failure_ab_add_participant
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:sadd).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_add_participant("price_options", "3ff62e2fb51f0b22646a342a2d357aec", 0)
+    end
+  end
+
+  def test_graceful_failure_ab_seen
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:sismember).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_seen("price_options", "3ff62e2fb51f0b22646a342a2d357aec", 0)
+    end
+  end
+
+  def test_graceful_failure_ab_assigned
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:sismember).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_assigned("price_options", "3ff62e2fb51f0b22646a342a2d357aec")
+    end
+  end
+
+  def test_graceful_failure_ab_add_conversion
+    redis_adapter, mocked_redis = stub_redis
+    mocked_redis.stubs(:sismember).raises(RuntimeError)
+
+    assert_nothing_raised do
+      redis_adapter.ab_add_conversion("price_options", 0, "3ff62e2fb51f0b22646a342a2d357aec")
+    end
+  end
+
 end

data/test/dummy/config/database.yml

@@ -1,5 +1,22 @@
+# Based on https://github.com/plataformatec/devise/blob/master/test/rails_app/config/database.yml
+
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+
+# Set databases in all environments since we may test loading Rails in non-
+# test environments
+
+development:
+  adapter: sqlite3
+  database: ":memory:"
+
 test:
-  adapter: mysql
-  database: vanity_test
-  pool: 5
-  timeout: 5000
+  adapter: sqlite3
+  database: ":memory:"
+
+production:
+  adapter: sqlite3
+  database: ":memory:"

data/test/dummy/config/routes.rb

@@ -54,5 +54,5 @@ Dummy::Application.routes.draw do
 
   # This is a legacy wild controller route that's not recommended for RESTful applications.
   # Note: This route will make all actions in every controller accessible via GET requests.
-  match ':controller(/:action(/:id(.:format)))', :via => [:get]
+  match ':controller(/:action(/:id(.:format)))', :via => [:get, :post]
 end

data/test/experiment/ab_test.rb

@@ -133,7 +133,7 @@ class AbTestTest < ActionController::TestCase
 
   # -- use_js! --
 
-  def test_does_not_record_participant_when_using_js
+  def test_choose_does_not_record_participant_when_using_js
     Vanity.playground.use_js!
     ids = (0...10).to_a
     new_ab_test :foobar do
@@ -160,6 +160,44 @@ class AbTestTest < ActionController::TestCase
     assert_equal 1, on_assignment_called_times
   end
 
+  def test_calls_on_assignment_when_given_valid_request
+    on_assignment_called_times = 0
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      metrics :coolness
+      on_assignment { on_assignment_called_times = on_assignment_called_times+1 }
+    end
+    experiment(:foobar).choose(dummy_request)
+    assert_equal 1, on_assignment_called_times
+  end
+
+  def test_does_not_call_on_assignment_when_given_invalid_request
+    on_assignment_called_times = 0
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      metrics :coolness
+      on_assignment { on_assignment_called_times = on_assignment_called_times+1 }
+    end
+    request = dummy_request
+    request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
+    experiment(:foobar).choose(request)
+    assert_equal 0, on_assignment_called_times
+  end
+
+  def test_calls_on_assignment_on_new_assignment_via_chooses
+    on_assignment_called_times = 0
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "6e98ec" }
+      metrics :coolness
+      on_assignment { on_assignment_called_times = on_assignment_called_times+1 }
+    end
+    2.times { experiment(:foobar).chooses("foo") }
+    assert_equal 1, on_assignment_called_times
+  end
+
   def test_returns_the_same_alternative_consistently_when_on_assignment_is_set
     new_ab_test :foobar do
       alternatives "foo", "bar"
@@ -344,6 +382,27 @@ class AbTestTest < ActionController::TestCase
     assert_equal 100, alts.map(&:converted).sum
   end
 
+  def test_choose_records_participants_given_a_valid_request
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "me" }
+      metrics :coolness
+    end
+    experiment(:foobar).choose(dummy_request)
+    assert_equal 1, experiment(:foobar).alternatives.map(&:participants).sum
+  end
+
+  def test_choose_ignores_participants_given_an_invalid_request
+    new_ab_test :foobar do
+      alternatives "foo", "bar"
+      identify { "me" }
+      metrics :coolness
+    end
+    request = dummy_request
+    request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
+    experiment(:foobar).choose(request)
+    assert_equal 0, experiment(:foobar).alternatives.map(&:participants).sum
+  end
 
   def test_destroy_experiment
     new_ab_test :simple do
@@ -465,6 +524,7 @@ class AbTestTest < ActionController::TestCase
 
 
   # -- Scoring --
+
   def test_calculate_score
     new_ab_test :abcd do
       alternatives :a, :b, :c, :d
@@ -488,10 +548,10 @@ class AbTestTest < ActionController::TestCase
       metrics :coolness
     end
     # participating, conversions, rate, z-score
-    # Control:      182	35 19.23%	N/A
-    # Treatment A:  180	45 25.00%	1.33
-    # treatment B:  189	28 14.81%	-1.13
-    # treatment C:  188	61 32.45%	2.94
+    # Control:      182 35 19.23% N/A
+    # Treatment A:  180 45 25.00% 1.33
+    # treatment B:  189 28 14.81% -1.13
+    # treatment C:  188 61 32.45% 2.94
     fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
 
     z_scores = experiment(:abcd).score.alts.map { |alt| "%.2f" % alt.z_score }
@@ -514,10 +574,10 @@ class AbTestTest < ActionController::TestCase
       metrics :coolness
     end
     # participating, conversions, rate, z-score
-    # Control:      182	35 19.23%	N/A
-    # Treatment A:  180	45 25.00%	1.33
-    # treatment B:  189	28 14.81%	-1.13
-    # treatment C:  188	61 32.45%	2.94
+    # Control:      182 35 19.23% N/A
+    # Treatment A:  180 45 25.00% 1.33
+    # treatment B:  189 28 14.81% -1.13
+    # treatment C:  188 61 32.45% 2.94
     fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
 
     score_result = experiment(:abcd).bayes_bandit_score
@@ -593,10 +653,10 @@ class AbTestTest < ActionController::TestCase
       metrics :coolness
     end
     # participating, conversions, rate, z-score
-    # Control:      182	35 19.23%	N/A
-    # Treatment A:  180	45 25.00%	1.33
-    # treatment B:  189	28 14.81%	-1.13
-    # treatment C:  188	61 32.45%	2.94
+    # Control:      182 35 19.23% N/A
+    # Treatment A:  180 45 25.00% 1.33
+    # treatment B:  189 28 14.81% -1.13
+    # treatment C:  188 61 32.45% 2.94
     fake :abcd, :a=>[182, 35], :b=>[180, 45], :c=>[189,28], :d=>[188, 61]
 
     assert_equal <<-TEXT, experiment(:abcd).conclusion.join("\n") << "\n"
@@ -919,6 +979,26 @@ This experiment did not run long enough to find a clear winner.
     assert_equal experiment(:simple).alternatives[2].participants, 1
   end
 
+  def test_chooses_records_participants_given_a_valid_request
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    experiment(:simple).chooses(:a, dummy_request)
+    assert_equal 1, experiment(:simple).alternatives[0].participants
+  end
+
+  def test_chooses_ignores_participants_given_an_invalid_request
+    new_ab_test :simple do
+      alternatives :a, :b, :c
+      metrics :coolness
+    end
+    request = dummy_request
+    request.user_agent = "Googlebot/2.1 ( http://www.google.com/bot.html)"
+    experiment(:simple).chooses(:a, request)
+    assert_equal 0, experiment(:simple).alternatives[0].participants
+  end
+
   def test_no_collection_and_chooses
     not_collecting!
     new_ab_test :simple do

data/test/metric/active_record_test.rb

@@ -1,10 +1,12 @@
 require "test/test_helper"
 
 class Sky < ActiveRecord::Base
-  connection.drop_table :skies if table_exists?
-  connection.create_table :skies do |t|
-    t.integer :height
-    t.timestamps
+  if connected?
+    connection.drop_table :skies if table_exists?
+    connection.create_table :skies do |t|
+      t.integer :height
+      t.timestamps
+    end
   end
 
   if defined?(Rails::Railtie)
@@ -14,6 +16,7 @@ class Sky < ActiveRecord::Base
   end
 end
 
+if ActiveRecord::Base.connected?
 
 context "ActiveRecord Metric" do
 
@@ -305,3 +308,5 @@ context "ActiveRecord Metric" do
     end
   end
 end
+
+end

data/test/passenger_test.rb

@@ -1,51 +1,52 @@
 require "test/test_helper"
 
-#not supported for rails3
-unless defined?(Rails::Railtie)
-require "phusion_passenger/spawn_manager"
-class PassengerTest < Test::Unit::TestCase
-  def setup
-    super
-    ActiveRecord::Base.connection.disconnect! # Otherwise AR metric tests fail
-    @original = Vanity.playground.connection
-    File.unlink "test/myapp/config/vanity.yml" rescue nil
-    File.open("test/myapp/config/vanity.yml", "w") do |io|
-      io.write YAML.dump({ "production"=>DATABASE })
+# Not supported for rails3
+if !defined?(Rails::Railtie) && ActiveRecord::Base.connected?
+  require "phusion_passenger/spawn_manager"
+
+  class PassengerTest < Test::Unit::TestCase
+    def setup
+      super
+      ActiveRecord::Base.connection.disconnect! # Otherwise AR metric tests fail
+      @original = Vanity.playground.connection
+      File.unlink "test/myapp/config/vanity.yml" rescue nil
+      File.open("test/myapp/config/vanity.yml", "w") do |io|
+	io.write YAML.dump({ "production"=>DATABASE })
+      end
+      @server = PhusionPassenger::SpawnManager.new
+      @server.start
+      Thread.pass until @server.started?
+      app_root = File.expand_path("myapp", File.dirname(__FILE__))
+      @app = @server.spawn_application "app_root"=>app_root, "spawn_method"=>"smart"
     end
-    @server = PhusionPassenger::SpawnManager.new
-    @server.start
-    Thread.pass until @server.started?
-    app_root = File.expand_path("myapp", File.dirname(__FILE__))
-    @app = @server.spawn_application "app_root"=>app_root, "spawn_method"=>"smart"
-  end
 
-  def test_reconnect
-    # When using AR adapter, we're not responsible to reconnect, and we're going
-    # to get the same "connect" (AR connection handler) either way.
-    # return if defined?(Vanity::Adapters::ActiveRecordAdapter) && Vanity::Adapters::ActiveRecordAdapter === Vanity.playground.connection
+    def test_reconnect
+      # When using AR adapter, we're not responsible to reconnect, and we're going
+      # to get the same "connect" (AR connection handler) either way.
+      # return if defined?(Vanity::Adapters::ActiveRecordAdapter) && Vanity::Adapters::ActiveRecordAdapter === Vanity.playground.connection
 
-    sleep 0.1
-    case @app.listen_socket_type
-    when "tcp" ; socket = TCPSocket.new(*@app.listen_socket_name.split(":"))
-    when "unix"; socket = UNIXSocket.new(@app.listen_socket_name)
-    else fail
+      sleep 0.1
+      case @app.listen_socket_type
+      when "tcp" ; socket = TCPSocket.new(*@app.listen_socket_name.split(":"))
+      when "unix"; socket = UNIXSocket.new(@app.listen_socket_name)
+      else fail
+      end
+      channel = PhusionPassenger::MessageChannel.new(socket)
+      request = {"REQUEST_PATH"=>"/", "REQUEST_METHOD"=>"GET", "QUERY_STRING"=>" "}
+      channel.write_scalar request.to_a.join("\0")
+      response = socket.read.split("\r\n\r\n").last
+      socket.close
+      conn, obj_id = response.split("\n")
+      assert_equal @original.to_s, conn
+      assert_not_equal @original.object_id.to_s, obj_id
     end
-    channel = PhusionPassenger::MessageChannel.new(socket)
-    request = {"REQUEST_PATH"=>"/", "REQUEST_METHOD"=>"GET", "QUERY_STRING"=>" "}
-    channel.write_scalar request.to_a.join("\0")
-    response = socket.read.split("\r\n\r\n").last
-    socket.close
-    conn, obj_id = response.split("\n")
-    assert_equal @original.to_s, conn
-    assert_not_equal @original.object_id.to_s, obj_id
-  end
 
-  def teardown
-    super
-    @server.cleanup
-    @server.stop
-    Process.kill('SIGKILL', @app.pid.to_i) # Just in case...KIDS, GET OUT OF THE POOL!
-    File.unlink "test/myapp/config/vanity.yml"
+    def teardown
+      super
+      @server.cleanup
+      @server.stop
+      Process.kill('SIGKILL', @app.pid.to_i) # Just in case...KIDS, GET OUT OF THE POOL!
+      File.unlink "test/myapp/config/vanity.yml"
+    end
   end
 end
-end